2024-11-21 00:26:10,024 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-21 00:26:10,069 main DEBUG Took 0.014385 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 00:26:10,070 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 00:26:10,070 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 00:26:10,071 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 00:26:10,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,082 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 00:26:10,097 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,099 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,100 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,101 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,102 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,105 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,106 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,107 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,108 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,110 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,111 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,112 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 00:26:10,113 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,114 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 00:26:10,116 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 00:26:10,118 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 00:26:10,121 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 00:26:10,121 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 00:26:10,123 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 00:26:10,124 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 00:26:10,136 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 00:26:10,140 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 00:26:10,142 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 00:26:10,144 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 00:26:10,145 main DEBUG createAppenders(={Console}) 2024-11-21 00:26:10,146 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-21 00:26:10,147 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-21 00:26:10,147 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-21 00:26:10,148 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 00:26:10,149 main DEBUG OutputStream closed 2024-11-21 00:26:10,149 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 00:26:10,149 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 00:26:10,150 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-21 00:26:10,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 00:26:10,260 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 00:26:10,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 00:26:10,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 00:26:10,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 00:26:10,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 00:26:10,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 00:26:10,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 00:26:10,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 00:26:10,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 00:26:10,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 00:26:10,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 00:26:10,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 00:26:10,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 00:26:10,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 00:26:10,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 00:26:10,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 00:26:10,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 00:26:10,274 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 00:26:10,275 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-21 00:26:10,275 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 00:26:10,276 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-21T00:26:10,607 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2 2024-11-21 00:26:10,613 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 00:26:10,613 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T00:26:10,625 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-21T00:26:10,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T00:26:10,678 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080, deleteOnExit=true 2024-11-21T00:26:10,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T00:26:10,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/test.cache.data in system properties and HBase conf 2024-11-21T00:26:10,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T00:26:10,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/hadoop.log.dir in system properties and HBase conf 2024-11-21T00:26:10,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T00:26:10,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T00:26:10,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T00:26:10,829 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T00:26:10,974 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T00:26:10,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T00:26:10,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T00:26:10,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T00:26:10,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T00:26:10,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T00:26:10,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T00:26:10,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T00:26:10,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T00:26:10,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T00:26:10,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/nfs.dump.dir in system properties and HBase conf 2024-11-21T00:26:10,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/java.io.tmpdir in system properties and HBase conf 2024-11-21T00:26:10,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T00:26:10,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T00:26:10,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T00:26:13,110 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T00:26:13,214 INFO [Time-limited test {}] log.Log(170): Logging initialized @4057ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T00:26:13,307 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T00:26:13,378 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T00:26:13,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T00:26:13,429 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T00:26:13,430 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T00:26:13,446 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T00:26:13,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/hadoop.log.dir/,AVAILABLE} 2024-11-21T00:26:13,451 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T00:26:13,689 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/java.io.tmpdir/jetty-localhost-37825-hadoop-hdfs-3_4_1-tests_jar-_-any-5977017593854875662/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T00:26:13,696 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37825} 2024-11-21T00:26:13,697 INFO [Time-limited test {}] server.Server(415): Started @4541ms 2024-11-21T00:26:14,478 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T00:26:14,491 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T00:26:14,500 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T00:26:14,501 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T00:26:14,501 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T00:26:14,502 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/hadoop.log.dir/,AVAILABLE} 2024-11-21T00:26:14,503 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T00:26:14,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/java.io.tmpdir/jetty-localhost-34775-hadoop-hdfs-3_4_1-tests_jar-_-any-12969386479199793513/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T00:26:14,643 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:34775} 2024-11-21T00:26:14,643 INFO [Time-limited test {}] server.Server(415): Started @5488ms 2024-11-21T00:26:14,710 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T00:26:15,859 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/data/data1/current/BP-1879054066-172.17.0.2-1732148771700/current, will proceed with Du for space computation calculation, 2024-11-21T00:26:15,859 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/data/data2/current/BP-1879054066-172.17.0.2-1732148771700/current, will proceed with Du for space computation calculation, 2024-11-21T00:26:15,927 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T00:26:15,978 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c9229e051a2c4d5 with lease ID 0x46cc62e9d201ede8: Processing first storage report for DS-f7829f00-4535-4106-9755-1351d40fc534 from datanode DatanodeRegistration(127.0.0.1:42153, datanodeUuid=8fa39e67-27c4-4fd4-9c72-99e352a1aa1d, infoPort=42921, infoSecurePort=0, ipcPort=37631, storageInfo=lv=-57;cid=testClusterID;nsid=1292596428;c=1732148771700) 2024-11-21T00:26:15,979 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c9229e051a2c4d5 with lease ID 0x46cc62e9d201ede8: from storage DS-f7829f00-4535-4106-9755-1351d40fc534 node DatanodeRegistration(127.0.0.1:42153, datanodeUuid=8fa39e67-27c4-4fd4-9c72-99e352a1aa1d, infoPort=42921, infoSecurePort=0, ipcPort=37631, storageInfo=lv=-57;cid=testClusterID;nsid=1292596428;c=1732148771700), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T00:26:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c9229e051a2c4d5 with lease ID 0x46cc62e9d201ede8: Processing first storage report for DS-e3c37e49-bbe8-4521-a2ff-939edd542cf0 from datanode DatanodeRegistration(127.0.0.1:42153, datanodeUuid=8fa39e67-27c4-4fd4-9c72-99e352a1aa1d, infoPort=42921, infoSecurePort=0, ipcPort=37631, storageInfo=lv=-57;cid=testClusterID;nsid=1292596428;c=1732148771700) 2024-11-21T00:26:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c9229e051a2c4d5 with lease ID 0x46cc62e9d201ede8: from storage DS-e3c37e49-bbe8-4521-a2ff-939edd542cf0 node DatanodeRegistration(127.0.0.1:42153, datanodeUuid=8fa39e67-27c4-4fd4-9c72-99e352a1aa1d, infoPort=42921, infoSecurePort=0, ipcPort=37631, storageInfo=lv=-57;cid=testClusterID;nsid=1292596428;c=1732148771700), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T00:26:16,010 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2 2024-11-21T00:26:16,109 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/zookeeper_0, clientPort=51011, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T00:26:16,119 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51011 2024-11-21T00:26:16,131 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:16,135 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741825_1001 (size=7) 2024-11-21T00:26:16,841 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 with version=8 2024-11-21T00:26:16,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/hbase-staging 2024-11-21T00:26:16,945 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T00:26:17,193 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5ed4808ef0e6:0 server-side Connection retries=45 2024-11-21T00:26:17,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:26:17,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T00:26:17,210 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T00:26:17,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:26:17,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T00:26:17,351 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T00:26:17,417 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T00:26:17,427 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T00:26:17,431 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T00:26:17,461 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28246 (auto-detected) 2024-11-21T00:26:17,462 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T00:26:17,484 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38537 2024-11-21T00:26:17,515 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38537 connecting to ZooKeeper ensemble=127.0.0.1:51011 2024-11-21T00:26:17,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385370x0, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T00:26:17,686 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38537-0x1015ac8c8430000 connected 2024-11-21T00:26:17,788 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:17,791 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:17,810 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:26:17,817 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29, hbase.cluster.distributed=false 2024-11-21T00:26:17,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T00:26:17,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38537 2024-11-21T00:26:17,904 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38537 2024-11-21T00:26:17,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38537 2024-11-21T00:26:17,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38537 2024-11-21T00:26:17,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38537 2024-11-21T00:26:18,139 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5ed4808ef0e6:0 server-side Connection retries=45 2024-11-21T00:26:18,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:26:18,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T00:26:18,142 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T00:26:18,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T00:26:18,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T00:26:18,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T00:26:18,150 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T00:26:18,151 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38589 2024-11-21T00:26:18,153 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38589 connecting to ZooKeeper ensemble=127.0.0.1:51011 2024-11-21T00:26:18,155 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:18,159 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:18,305 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385890x0, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T00:26:18,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38589-0x1015ac8c8430001 connected 2024-11-21T00:26:18,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:26:18,310 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T00:26:18,319 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T00:26:18,323 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T00:26:18,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T00:26:18,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38589 2024-11-21T00:26:18,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38589 2024-11-21T00:26:18,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38589 2024-11-21T00:26:18,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38589 2024-11-21T00:26:18,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38589 2024-11-21T00:26:18,365 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5ed4808ef0e6:38537 2024-11-21T00:26:18,368 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:18,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:26:18,386 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:26:18,387 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:18,420 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T00:26:18,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:18,421 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:18,422 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T00:26:18,423 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5ed4808ef0e6,38537,1732148777004 from backup master directory 2024-11-21T00:26:18,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:18,431 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:26:18,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T00:26:18,432 WARN [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T00:26:18,432 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:18,434 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T00:26:18,435 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T00:26:18,502 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/hbase.id] with ID: a1f9997a-520c-400d-b987-9988e298295c 2024-11-21T00:26:18,502 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/.tmp/hbase.id 2024-11-21T00:26:18,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741826_1002 (size=42) 2024-11-21T00:26:18,933 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/.tmp/hbase.id]:[hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/hbase.id] 2024-11-21T00:26:18,979 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T00:26:18,985 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T00:26:19,007 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-21T00:26:19,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:19,020 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:19,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741827_1003 (size=196) 2024-11-21T00:26:19,458 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:26:19,462 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T00:26:19,483 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:19,488 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T00:26:19,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741828_1004 (size=1189) 2024-11-21T00:26:19,964 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store 2024-11-21T00:26:19,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741829_1005 (size=34) 2024-11-21T00:26:20,391 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T00:26:20,394 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:20,396 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T00:26:20,396 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:26:20,397 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:26:20,398 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T00:26:20,399 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:26:20,399 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:26:20,400 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732148780396Disabling compacts and flushes for region at 1732148780396Disabling writes for close at 1732148780398 (+2 ms)Writing region close event to WAL at 1732148780399 (+1 ms)Closed at 1732148780399 2024-11-21T00:26:20,404 WARN [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/.initializing 2024-11-21T00:26:20,404 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/WALs/5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:20,414 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T00:26:20,430 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5ed4808ef0e6%2C38537%2C1732148777004, suffix=, logDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/WALs/5ed4808ef0e6,38537,1732148777004, archiveDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/oldWALs, maxLogs=10 2024-11-21T00:26:20,459 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/WALs/5ed4808ef0e6,38537,1732148777004/5ed4808ef0e6%2C38537%2C1732148777004.1732148780435, exclude list is [], retry=0 2024-11-21T00:26:20,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42153,DS-f7829f00-4535-4106-9755-1351d40fc534,DISK] 2024-11-21T00:26:20,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T00:26:20,527 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/WALs/5ed4808ef0e6,38537,1732148777004/5ed4808ef0e6%2C38537%2C1732148777004.1732148780435 2024-11-21T00:26:20,528 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42921:42921)] 2024-11-21T00:26:20,528 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:26:20,529 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:20,533 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,534 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T00:26:20,614 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:20,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:20,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T00:26:20,626 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:20,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:20,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T00:26:20,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:20,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:20,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T00:26:20,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:20,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:20,640 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,644 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,646 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,653 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,654 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,658 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T00:26:20,662 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T00:26:20,668 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:26:20,669 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61569445, jitterRate=-0.08254377543926239}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T00:26:20,678 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732148780547Initializing all the Stores at 1732148780550 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148780551 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148780551Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148780552 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148780552Cleaning up temporary data from old regions at 1732148780654 (+102 ms)Region opened successfully at 1732148780677 (+23 ms) 2024-11-21T00:26:20,679 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T00:26:20,720 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9000e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5ed4808ef0e6/172.17.0.2:0 2024-11-21T00:26:20,760 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T00:26:20,773 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T00:26:20,774 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T00:26:20,777 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T00:26:20,778 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T00:26:20,784 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T00:26:20,785 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T00:26:20,815 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T00:26:20,829 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T00:26:20,841 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T00:26:20,845 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T00:26:20,846 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T00:26:20,862 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T00:26:20,865 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T00:26:20,869 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T00:26:20,883 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T00:26:20,885 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T00:26:20,904 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T00:26:20,927 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T00:26:20,946 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T00:26:20,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T00:26:20,968 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T00:26:20,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:20,968 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:20,970 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5ed4808ef0e6,38537,1732148777004, sessionid=0x1015ac8c8430000, setting cluster-up flag (Was=false) 2024-11-21T00:26:21,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:21,057 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:21,168 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T00:26:21,170 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:21,210 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:21,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:21,273 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T00:26:21,275 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:21,282 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T00:26:21,356 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(746): ClusterId : a1f9997a-520c-400d-b987-9988e298295c 2024-11-21T00:26:21,359 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T00:26:21,366 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T00:26:21,370 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T00:26:21,370 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T00:26:21,377 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T00:26:21,379 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T00:26:21,380 DEBUG [RS:0;5ed4808ef0e6:38589 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cce369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5ed4808ef0e6/172.17.0.2:0 2024-11-21T00:26:21,384 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T00:26:21,390 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5ed4808ef0e6,38537,1732148777004 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T00:26:21,399 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:26:21,399 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:26:21,400 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:26:21,400 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5ed4808ef0e6:38589 2024-11-21T00:26:21,400 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T00:26:21,400 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5ed4808ef0e6:0, corePoolSize=10, maxPoolSize=10 2024-11-21T00:26:21,400 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,400 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T00:26:21,401 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,404 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T00:26:21,404 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T00:26:21,404 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T00:26:21,407 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(2659): reportForDuty to master=5ed4808ef0e6,38537,1732148777004 with port=38589, startcode=1732148778087 2024-11-21T00:26:21,413 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732148811412 2024-11-21T00:26:21,415 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T00:26:21,415 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T00:26:21,416 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T00:26:21,416 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T00:26:21,420 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T00:26:21,420 DEBUG [RS:0;5ed4808ef0e6:38589 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T00:26:21,420 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T00:26:21,421 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T00:26:21,421 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T00:26:21,422 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,425 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:21,426 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T00:26:21,440 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T00:26:21,442 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T00:26:21,442 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T00:26:21,452 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T00:26:21,453 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T00:26:21,468 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.large.0-1732148781454,5,FailOnTimeoutGroup] 2024-11-21T00:26:21,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741831_1007 (size=1321) 2024-11-21T00:26:21,488 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.small.0-1732148781468,5,FailOnTimeoutGroup] 2024-11-21T00:26:21,489 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,489 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T00:26:21,491 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,491 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,491 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T00:26:21,502 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38537 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:21,504 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38537 {}] master.ServerManager(517): Registering regionserver=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:21,521 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:26:21,521 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45671 2024-11-21T00:26:21,521 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T00:26:21,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T00:26:21,536 DEBUG [RS:0;5ed4808ef0e6:38589 {}] zookeeper.ZKUtil(111): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:21,537 WARN [RS:0;5ed4808ef0e6:38589 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T00:26:21,537 INFO [RS:0;5ed4808ef0e6:38589 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T00:26:21,537 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:21,539 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5ed4808ef0e6,38589,1732148778087] 2024-11-21T00:26:21,565 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T00:26:21,585 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T00:26:21,590 INFO [RS:0;5ed4808ef0e6:38589 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T00:26:21,591 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,594 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T00:26:21,601 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T00:26:21,603 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,603 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,603 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,603 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,604 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,604 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,604 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5ed4808ef0e6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T00:26:21,604 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,604 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,604 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,605 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,605 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,605 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5ed4808ef0e6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T00:26:21,605 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5ed4808ef0e6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T00:26:21,605 DEBUG [RS:0;5ed4808ef0e6:38589 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T00:26:21,609 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,609 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,609 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,609 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,609 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,609 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38589,1732148778087-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T00:26:21,633 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T00:26:21,635 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38589,1732148778087-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,636 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,636 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.Replication(171): 5ed4808ef0e6,38589,1732148778087 started 2024-11-21T00:26:21,656 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:21,656 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1482): Serving as 5ed4808ef0e6,38589,1732148778087, RpcServer on 5ed4808ef0e6/172.17.0.2:38589, sessionid=0x1015ac8c8430001 2024-11-21T00:26:21,657 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T00:26:21,658 DEBUG [RS:0;5ed4808ef0e6:38589 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:21,658 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5ed4808ef0e6,38589,1732148778087' 2024-11-21T00:26:21,658 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T00:26:21,659 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T00:26:21,660 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T00:26:21,660 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T00:26:21,660 DEBUG [RS:0;5ed4808ef0e6:38589 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:21,660 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5ed4808ef0e6,38589,1732148778087' 2024-11-21T00:26:21,660 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T00:26:21,661 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T00:26:21,662 DEBUG [RS:0;5ed4808ef0e6:38589 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T00:26:21,662 INFO [RS:0;5ed4808ef0e6:38589 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T00:26:21,662 INFO [RS:0;5ed4808ef0e6:38589 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T00:26:21,768 INFO [RS:0;5ed4808ef0e6:38589 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T00:26:21,772 INFO [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5ed4808ef0e6%2C38589%2C1732148778087, suffix=, logDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087, archiveDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/oldWALs, maxLogs=32 2024-11-21T00:26:21,790 DEBUG [RS:0;5ed4808ef0e6:38589 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087/5ed4808ef0e6%2C38589%2C1732148778087.1732148781775, exclude list is [], retry=0 2024-11-21T00:26:21,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42153,DS-f7829f00-4535-4106-9755-1351d40fc534,DISK] 2024-11-21T00:26:21,798 INFO [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087/5ed4808ef0e6%2C38589%2C1732148778087.1732148781775 2024-11-21T00:26:21,799 DEBUG [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42921:42921)] 2024-11-21T00:26:21,886 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T00:26:21,886 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:26:21,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741833_1009 (size=32) 2024-11-21T00:26:22,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:22,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T00:26:22,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T00:26:22,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T00:26:22,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T00:26:22,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T00:26:22,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T00:26:22,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T00:26:22,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T00:26:22,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T00:26:22,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740 2024-11-21T00:26:22,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740 2024-11-21T00:26:22,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T00:26:22,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T00:26:22,327 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:26:22,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T00:26:22,334 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:26:22,335 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63058450, jitterRate=-0.06035587191581726}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:26:22,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732148782298Initializing all the Stores at 1732148782300 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148782300Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148782300Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148782301 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148782301Cleaning up temporary data from old regions at 1732148782326 (+25 ms)Region opened successfully at 1732148782339 (+13 ms) 2024-11-21T00:26:22,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T00:26:22,339 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T00:26:22,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T00:26:22,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T00:26:22,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T00:26:22,341 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T00:26:22,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732148782339Disabling compacts and flushes for region at 1732148782339Disabling writes for close at 1732148782339Writing region close event to WAL at 1732148782340 (+1 ms)Closed at 1732148782341 (+1 ms) 2024-11-21T00:26:22,344 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T00:26:22,344 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T00:26:22,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T00:26:22,361 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T00:26:22,364 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T00:26:22,517 DEBUG [5ed4808ef0e6:38537 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T00:26:22,529 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:22,536 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5ed4808ef0e6,38589,1732148778087, state=OPENING 2024-11-21T00:26:22,553 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T00:26:22,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:22,564 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:26:22,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:26:22,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:26:22,566 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T00:26:22,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:26:22,741 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T00:26:22,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46651, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T00:26:22,758 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T00:26:22,759 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T00:26:22,760 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T00:26:22,766 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5ed4808ef0e6%2C38589%2C1732148778087.meta, suffix=.meta, logDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087, archiveDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/oldWALs, maxLogs=32 2024-11-21T00:26:22,787 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087/5ed4808ef0e6%2C38589%2C1732148778087.meta.1732148782769.meta, exclude list is [], retry=0 2024-11-21T00:26:22,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42153,DS-f7829f00-4535-4106-9755-1351d40fc534,DISK] 2024-11-21T00:26:22,809 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087/5ed4808ef0e6%2C38589%2C1732148778087.meta.1732148782769.meta 2024-11-21T00:26:22,816 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42921:42921)] 2024-11-21T00:26:22,817 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:26:22,819 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T00:26:22,824 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T00:26:22,830 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T00:26:22,835 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T00:26:22,836 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:22,836 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T00:26:22,836 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T00:26:22,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T00:26:22,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T00:26:22,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T00:26:22,847 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T00:26:22,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T00:26:22,851 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T00:26:22,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T00:26:22,855 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T00:26:22,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:22,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T00:26:22,857 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T00:26:22,858 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740 2024-11-21T00:26:22,862 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740 2024-11-21T00:26:22,865 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T00:26:22,865 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T00:26:22,866 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:26:22,869 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T00:26:22,871 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73256907, jitterRate=0.09161297976970673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:26:22,871 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T00:26:22,873 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732148782837Writing region info on filesystem at 1732148782837Initializing all the Stores at 1732148782839 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148782839Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148782840 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148782840Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732148782840Cleaning up temporary data from old regions at 1732148782865 (+25 ms)Running coprocessor post-open hooks at 1732148782872 (+7 ms)Region opened successfully at 1732148782873 (+1 ms) 2024-11-21T00:26:22,882 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732148782733 2024-11-21T00:26:22,894 DEBUG [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T00:26:22,895 INFO [RS_OPEN_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T00:26:22,896 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:22,899 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5ed4808ef0e6,38589,1732148778087, state=OPEN 2024-11-21T00:26:22,934 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T00:26:22,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T00:26:22,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:26:22,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T00:26:22,935 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:22,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T00:26:22,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5ed4808ef0e6,38589,1732148778087 in 368 msec 2024-11-21T00:26:22,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T00:26:22,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 592 msec 2024-11-21T00:26:22,951 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T00:26:22,951 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T00:26:22,974 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:22,976 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:22,996 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:22,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54385, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:23,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7060 sec 2024-11-21T00:26:23,023 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732148783023, completionTime=-1 2024-11-21T00:26:23,026 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T00:26:23,026 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-21T00:26:23,079 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-21T00:26:23,079 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732148843079 2024-11-21T00:26:23,079 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732148903079 2024-11-21T00:26:23,079 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 53 msec 2024-11-21T00:26:23,082 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38537,1732148777004-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,083 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38537,1732148777004-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,083 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38537,1732148777004-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,088 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5ed4808ef0e6:38537, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,089 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,090 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,104 DEBUG [master/5ed4808ef0e6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T00:26:23,131 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.698sec 2024-11-21T00:26:23,132 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T00:26:23,133 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T00:26:23,134 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T00:26:23,135 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T00:26:23,135 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T00:26:23,136 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38537,1732148777004-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T00:26:23,136 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38537,1732148777004-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T00:26:23,144 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T00:26:23,145 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T00:26:23,145 INFO [master/5ed4808ef0e6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5ed4808ef0e6,38537,1732148777004-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T00:26:23,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfad15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:23,189 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T00:26:23,189 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T00:26:23,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:23,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:23,203 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:23,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:23,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:23,215 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:23,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b5c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:23,218 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:23,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:23,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:23,268 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=645, ProcessCount=11, AvailableMemoryMB=1963 2024-11-21T00:26:23,277 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:26:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008 2024-11-21T00:26:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:26:23,295 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:26:23,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:26:23,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:26:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:23,316 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:26:23,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T00:26:23,319 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:23,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:26:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:26:23,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741835_1011 (size=959) 2024-11-21T00:26:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:26:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:26:23,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:26:23,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741836_1012 (size=53) 2024-11-21T00:26:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:26:24,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:24,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b03d35ead3c53e10e140da78ec4bd637, disabling compactions & flushes 2024-11-21T00:26:24,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. after waiting 0 ms 2024-11-21T00:26:24,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b03d35ead3c53e10e140da78ec4bd637: Waiting for close lock at 1732148784158Disabling compacts and flushes for region at 1732148784158Disabling writes for close at 1732148784158Writing region close event to WAL at 1732148784158Closed at 1732148784158 2024-11-21T00:26:24,161 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:26:24,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148784161"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148784161"}]},"ts":"1732148784161"} 2024-11-21T00:26:24,172 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:26:24,174 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:26:24,177 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148784174"}]},"ts":"1732148784174"} 2024-11-21T00:26:24,182 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:26:24,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, ASSIGN}] 2024-11-21T00:26:24,188 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, ASSIGN 2024-11-21T00:26:24,190 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=false 2024-11-21T00:26:24,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b03d35ead3c53e10e140da78ec4bd637, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:24,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, ASSIGN because future has completed 2024-11-21T00:26:24,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:26:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:26:24,510 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,511 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:26:24,511 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,511 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:24,512 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,512 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,515 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,517 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b03d35ead3c53e10e140da78ec4bd637 columnFamilyName A 2024-11-21T00:26:24,518 DEBUG [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:24,529 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] regionserver.HStore(327): Store=b03d35ead3c53e10e140da78ec4bd637/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:24,529 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,534 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b03d35ead3c53e10e140da78ec4bd637 columnFamilyName B 2024-11-21T00:26:24,534 DEBUG [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:24,535 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] regionserver.HStore(327): Store=b03d35ead3c53e10e140da78ec4bd637/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:24,536 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,539 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b03d35ead3c53e10e140da78ec4bd637 columnFamilyName C 2024-11-21T00:26:24,539 DEBUG [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:24,540 INFO [StoreOpener-b03d35ead3c53e10e140da78ec4bd637-1 {}] regionserver.HStore(327): Store=b03d35ead3c53e10e140da78ec4bd637/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:24,541 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,543 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,544 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,551 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,552 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,553 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:26:24,557 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,564 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:26:24,566 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b03d35ead3c53e10e140da78ec4bd637; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70658602, jitterRate=0.05289521813392639}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:26:24,566 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:24,568 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b03d35ead3c53e10e140da78ec4bd637: Running coprocessor pre-open hook at 1732148784512Writing region info on filesystem at 1732148784512Initializing all the Stores at 1732148784514 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148784514Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148784514Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148784514Cleaning up temporary data from old regions at 1732148784552 (+38 ms)Running coprocessor post-open hooks at 1732148784566 (+14 ms)Region opened successfully at 1732148784568 (+2 ms) 2024-11-21T00:26:24,571 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., pid=6, masterSystemTime=1732148784502 2024-11-21T00:26:24,580 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,580 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:24,583 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b03d35ead3c53e10e140da78ec4bd637, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:24,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:26:24,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T00:26:24,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 in 249 msec 2024-11-21T00:26:24,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T00:26:24,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, ASSIGN in 424 msec 2024-11-21T00:26:24,619 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:26:24,620 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148784619"}]},"ts":"1732148784619"} 2024-11-21T00:26:24,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:26:24,627 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:26:24,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3210 sec 2024-11-21T00:26:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T00:26:25,485 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:25,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cf95fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbd9518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,526 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aaf91b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,529 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:26:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4014441b 2024-11-21T00:26:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:26:25,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:26:25,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71cbc6ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecdd502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5376a17a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71374046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,598 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d08ce54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,610 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e463f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760c0989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c44f32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17daddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57135afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3ac0b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,669 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c90417c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a75e5f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443701ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4519e7b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,702 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2b44cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:25,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:25,710 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276c4225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:25,714 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf634c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:25,728 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,732 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,745 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:25,750 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,753 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,756 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,760 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,765 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,767 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:25,767 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,768 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T00:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:25,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:25,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,793 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:25,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:25,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:25,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:26:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:25,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:26:26,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:26:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:26,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/bc651fc895ff408dad69028107d19181 is 50, key is test_row_0/A:col10/1732148785846/Put/seqid=0 2024-11-21T00:26:26,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741837_1013 (size=16681) 2024-11-21T00:26:26,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/bc651fc895ff408dad69028107d19181 2024-11-21T00:26:26,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:26:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148846183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148846183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148846198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148846195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148846198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148846374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:26:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148846379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148846379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148846379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148846384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:26,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/8038f3c60a4c47a99e962a21e2a611c5 is 50, key is test_row_0/B:col10/1732148785846/Put/seqid=0 2024-11-21T00:26:26,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741838_1014 (size=12001) 2024-11-21T00:26:26,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/8038f3c60a4c47a99e962a21e2a611c5 2024-11-21T00:26:26,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/25842dcd72684aba84547dadbeed323b is 50, key is test_row_0/C:col10/1732148785846/Put/seqid=0 2024-11-21T00:26:26,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:26:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741839_1015 (size=12001) 2024-11-21T00:26:26,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/25842dcd72684aba84547dadbeed323b 2024-11-21T00:26:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148846594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148846605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148846606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148846606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148846606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:26,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/bc651fc895ff408dad69028107d19181 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/bc651fc895ff408dad69028107d19181 2024-11-21T00:26:26,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/bc651fc895ff408dad69028107d19181, entries=250, sequenceid=13, filesize=16.3 K 2024-11-21T00:26:26,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/8038f3c60a4c47a99e962a21e2a611c5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8038f3c60a4c47a99e962a21e2a611c5 2024-11-21T00:26:26,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8038f3c60a4c47a99e962a21e2a611c5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:26:26,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/25842dcd72684aba84547dadbeed323b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/25842dcd72684aba84547dadbeed323b 2024-11-21T00:26:26,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:26:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:26,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/25842dcd72684aba84547dadbeed323b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:26:26,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b03d35ead3c53e10e140da78ec4bd637 in 857ms, sequenceid=13, compaction requested=false 2024-11-21T00:26:26,762 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T00:26:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:26,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T00:26:26,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:26,891 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T00:26:26,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/27d3ad216e5242c6b7a409582675e683 is 50, key is test_row_0/A:col10/1732148786193/Put/seqid=0 2024-11-21T00:26:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:26,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741840_1016 (size=12001) 2024-11-21T00:26:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148846969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148846995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148847003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148846995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148847006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148847113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148847156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148847156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148847157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148847157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148847343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,381 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/27d3ad216e5242c6b7a409582675e683 2024-11-21T00:26:27,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148847373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148847374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148847375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148847375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,413 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T00:26:27,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T00:26:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/d1fa11e5d5d2410d92ccd795512682ea is 50, key is test_row_0/B:col10/1732148786193/Put/seqid=0 2024-11-21T00:26:27,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T00:26:27,423 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T00:26:27,424 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T00:26:27,424 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T00:26:27,427 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T00:26:27,427 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T00:26:27,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741841_1017 (size=12001) 2024-11-21T00:26:27,559 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T00:26:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148847664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148847693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148847704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148847704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148847714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:27,886 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/d1fa11e5d5d2410d92ccd795512682ea 2024-11-21T00:26:27,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d8e8237faed64c71bc28a099871f8c4c is 50, key is test_row_0/C:col10/1732148786193/Put/seqid=0 2024-11-21T00:26:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:27,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741842_1018 (size=12001) 2024-11-21T00:26:27,955 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d8e8237faed64c71bc28a099871f8c4c 2024-11-21T00:26:27,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/27d3ad216e5242c6b7a409582675e683 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/27d3ad216e5242c6b7a409582675e683 2024-11-21T00:26:27,987 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/27d3ad216e5242c6b7a409582675e683, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T00:26:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/d1fa11e5d5d2410d92ccd795512682ea as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/d1fa11e5d5d2410d92ccd795512682ea 2024-11-21T00:26:28,027 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/d1fa11e5d5d2410d92ccd795512682ea, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T00:26:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d8e8237faed64c71bc28a099871f8c4c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d8e8237faed64c71bc28a099871f8c4c 2024-11-21T00:26:28,060 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d8e8237faed64c71bc28a099871f8c4c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T00:26:28,063 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for b03d35ead3c53e10e140da78ec4bd637 in 1173ms, sequenceid=37, compaction requested=false 2024-11-21T00:26:28,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:28,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:28,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T00:26:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T00:26:28,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T00:26:28,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2680 sec 2024-11-21T00:26:28,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3270 sec 2024-11-21T00:26:28,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:28,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T00:26:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:28,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/964c40e5f8ee40d0afa021ac00977ec3 is 50, key is test_row_0/A:col10/1732148786971/Put/seqid=0 2024-11-21T00:26:28,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741843_1019 (size=12001) 2024-11-21T00:26:28,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/964c40e5f8ee40d0afa021ac00977ec3 2024-11-21T00:26:28,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148848300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148848304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148848315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148848316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148848322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/ea582647b9aa4d479b600faeb3fb7b53 is 50, key is test_row_0/B:col10/1732148786971/Put/seqid=0 2024-11-21T00:26:28,405 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:26:28,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741844_1020 (size=12001) 2024-11-21T00:26:28,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/ea582647b9aa4d479b600faeb3fb7b53 2024-11-21T00:26:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148848433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148848433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148848443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148848444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148848444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d21a3ce691d842e2b229815c2aaae993 is 50, key is test_row_0/C:col10/1732148786971/Put/seqid=0 2024-11-21T00:26:28,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741845_1021 (size=12001) 2024-11-21T00:26:28,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d21a3ce691d842e2b229815c2aaae993 2024-11-21T00:26:28,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/964c40e5f8ee40d0afa021ac00977ec3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/964c40e5f8ee40d0afa021ac00977ec3 2024-11-21T00:26:28,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/964c40e5f8ee40d0afa021ac00977ec3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:26:28,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/ea582647b9aa4d479b600faeb3fb7b53 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ea582647b9aa4d479b600faeb3fb7b53 2024-11-21T00:26:28,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ea582647b9aa4d479b600faeb3fb7b53, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:26:28,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d21a3ce691d842e2b229815c2aaae993 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d21a3ce691d842e2b229815c2aaae993 2024-11-21T00:26:28,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d21a3ce691d842e2b229815c2aaae993, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:26:28,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b03d35ead3c53e10e140da78ec4bd637 in 455ms, sequenceid=51, compaction requested=true 2024-11-21T00:26:28,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:28,681 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:28,681 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:28,685 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:28,685 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:28,687 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:28,687 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:28,688 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:28,688 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:28,688 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8038f3c60a4c47a99e962a21e2a611c5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/d1fa11e5d5d2410d92ccd795512682ea, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ea582647b9aa4d479b600faeb3fb7b53] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.2 K 2024-11-21T00:26:28,688 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/bc651fc895ff408dad69028107d19181, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/27d3ad216e5242c6b7a409582675e683, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/964c40e5f8ee40d0afa021ac00977ec3] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=39.7 K 2024-11-21T00:26:28,691 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc651fc895ff408dad69028107d19181, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148785831 2024-11-21T00:26:28,691 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8038f3c60a4c47a99e962a21e2a611c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148785846 2024-11-21T00:26:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:28,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:28,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T00:26:28,695 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d1fa11e5d5d2410d92ccd795512682ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732148786175 2024-11-21T00:26:28,697 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ea582647b9aa4d479b600faeb3fb7b53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148786971 2024-11-21T00:26:28,697 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d3ad216e5242c6b7a409582675e683, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732148786175 2024-11-21T00:26:28,699 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 964c40e5f8ee40d0afa021ac00977ec3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148786971 2024-11-21T00:26:28,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/f82e3e0acd744e4bbc2d17568bf8e863 is 50, key is test_row_0/A:col10/1732148788685/Put/seqid=0 2024-11-21T00:26:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148848740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148848743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148848747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148848755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148848765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,795 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#10 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:28,796 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/5f536cb5ee4e4d26b9b097fb63b48574 is 50, key is test_row_0/B:col10/1732148786971/Put/seqid=0 2024-11-21T00:26:28,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741846_1022 (size=12001) 2024-11-21T00:26:28,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/f82e3e0acd744e4bbc2d17568bf8e863 2024-11-21T00:26:28,810 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#11 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:28,811 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/1350a93c5d8b4469ba5d1206d87b1239 is 50, key is test_row_0/A:col10/1732148786971/Put/seqid=0 2024-11-21T00:26:28,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741847_1023 (size=12104) 2024-11-21T00:26:28,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/066a5683a8664e2887aa63b15655dfb0 is 50, key is test_row_0/B:col10/1732148788685/Put/seqid=0 2024-11-21T00:26:28,880 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/5f536cb5ee4e4d26b9b097fb63b48574 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/5f536cb5ee4e4d26b9b097fb63b48574 2024-11-21T00:26:28,906 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into 5f536cb5ee4e4d26b9b097fb63b48574(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:28,906 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:28,912 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=13, startTime=1732148788679; duration=0sec 2024-11-21T00:26:28,913 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:28,913 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:28,914 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:28,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741848_1024 (size=12104) 2024-11-21T00:26:28,918 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:28,923 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:28,923 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:28,923 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/25842dcd72684aba84547dadbeed323b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d8e8237faed64c71bc28a099871f8c4c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d21a3ce691d842e2b229815c2aaae993] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.2 K 2024-11-21T00:26:28,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148848876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148848878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,925 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 25842dcd72684aba84547dadbeed323b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148785846 2024-11-21T00:26:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,929 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/1350a93c5d8b4469ba5d1206d87b1239 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/1350a93c5d8b4469ba5d1206d87b1239 2024-11-21T00:26:28,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148848879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148848882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:28,937 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e8237faed64c71bc28a099871f8c4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732148786175 2024-11-21T00:26:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148848892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:28,940 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d21a3ce691d842e2b229815c2aaae993, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148786971 2024-11-21T00:26:28,943 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into 1350a93c5d8b4469ba5d1206d87b1239(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:28,944 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:28,944 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=13, startTime=1732148788672; duration=0sec 2024-11-21T00:26:28,945 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:28,945 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:28,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741849_1025 (size=12001) 2024-11-21T00:26:28,975 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:28,976 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/6bd57be4522f4c08bda918cda990c9a6 is 50, key is test_row_0/C:col10/1732148786971/Put/seqid=0 2024-11-21T00:26:29,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741850_1026 (size=12104) 2024-11-21T00:26:29,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148849133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148849133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148849143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148849143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148849144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/066a5683a8664e2887aa63b15655dfb0 2024-11-21T00:26:29,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/e46f28e57000460d8ab8dfd2c6d4052a is 50, key is test_row_0/C:col10/1732148788685/Put/seqid=0 2024-11-21T00:26:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741851_1027 (size=12001) 2024-11-21T00:26:29,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/e46f28e57000460d8ab8dfd2c6d4052a 2024-11-21T00:26:29,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148849453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148849453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/f82e3e0acd744e4bbc2d17568bf8e863 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f82e3e0acd744e4bbc2d17568bf8e863 2024-11-21T00:26:29,469 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/6bd57be4522f4c08bda918cda990c9a6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/6bd57be4522f4c08bda918cda990c9a6 2024-11-21T00:26:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148849462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f82e3e0acd744e4bbc2d17568bf8e863, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T00:26:29,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/066a5683a8664e2887aa63b15655dfb0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/066a5683a8664e2887aa63b15655dfb0 2024-11-21T00:26:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148849473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148849473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:29,500 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into 6bd57be4522f4c08bda918cda990c9a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:29,500 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:29,501 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=13, startTime=1732148788679; duration=0sec 2024-11-21T00:26:29,503 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:29,503 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:29,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/066a5683a8664e2887aa63b15655dfb0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T00:26:29,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/e46f28e57000460d8ab8dfd2c6d4052a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/e46f28e57000460d8ab8dfd2c6d4052a 2024-11-21T00:26:29,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/e46f28e57000460d8ab8dfd2c6d4052a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T00:26:29,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for b03d35ead3c53e10e140da78ec4bd637 in 838ms, sequenceid=76, compaction requested=false 2024-11-21T00:26:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T00:26:29,944 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:29,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T00:26:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:26:29,949 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:29,951 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:29,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:26:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:30,061 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:30,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:26:30,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/60a39cae9f1d462495699ec327f1fee3 is 50, key is test_row_0/A:col10/1732148789970/Put/seqid=0 2024-11-21T00:26:30,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:30,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:30,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741852_1028 (size=12001) 2024-11-21T00:26:30,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148850218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148850233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:26:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148850245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148850252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148850269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148850374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148850375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148850394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148850394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148850414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/60a39cae9f1d462495699ec327f1fee3 2024-11-21T00:26:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:26:30,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/4065282fffb647c8aaddfb5d6a0c92b9 is 50, key is test_row_0/B:col10/1732148789970/Put/seqid=0 2024-11-21T00:26:30,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148850593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148850593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:30,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148850603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148850604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148850634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741853_1029 (size=12001) 2024-11-21T00:26:30,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/4065282fffb647c8aaddfb5d6a0c92b9 2024-11-21T00:26:30,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/2d4483b301b045de9df3eee3d4930f75 is 50, key is test_row_0/C:col10/1732148789970/Put/seqid=0 2024-11-21T00:26:30,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741854_1030 (size=12001) 2024-11-21T00:26:30,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:30,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:30,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148850912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148850913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:30,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148850944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148850949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:30,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148850943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:26:31,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:31,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:31,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/2d4483b301b045de9df3eee3d4930f75 2024-11-21T00:26:31,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/60a39cae9f1d462495699ec327f1fee3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/60a39cae9f1d462495699ec327f1fee3 2024-11-21T00:26:31,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/60a39cae9f1d462495699ec327f1fee3, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T00:26:31,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/4065282fffb647c8aaddfb5d6a0c92b9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/4065282fffb647c8aaddfb5d6a0c92b9 2024-11-21T00:26:31,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/4065282fffb647c8aaddfb5d6a0c92b9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T00:26:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/2d4483b301b045de9df3eee3d4930f75 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/2d4483b301b045de9df3eee3d4930f75 2024-11-21T00:26:31,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:31,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/2d4483b301b045de9df3eee3d4930f75, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T00:26:31,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for b03d35ead3c53e10e140da78ec4bd637 in 1226ms, sequenceid=94, compaction requested=true 2024-11-21T00:26:31,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:31,288 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:31,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:31,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:31,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:31,291 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:31,291 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:31,291 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:31,291 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,291 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/1350a93c5d8b4469ba5d1206d87b1239, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f82e3e0acd744e4bbc2d17568bf8e863, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/60a39cae9f1d462495699ec327f1fee3] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.3 K 2024-11-21T00:26:31,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:31,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:31,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:31,293 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:31,293 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1350a93c5d8b4469ba5d1206d87b1239, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148786971 2024-11-21T00:26:31,293 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:31,293 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,293 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/5f536cb5ee4e4d26b9b097fb63b48574, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/066a5683a8664e2887aa63b15655dfb0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/4065282fffb647c8aaddfb5d6a0c92b9] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.3 K 2024-11-21T00:26:31,296 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f82e3e0acd744e4bbc2d17568bf8e863, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732148788675 2024-11-21T00:26:31,296 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f536cb5ee4e4d26b9b097fb63b48574, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148786971 2024-11-21T00:26:31,296 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60a39cae9f1d462495699ec327f1fee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732148788731 2024-11-21T00:26:31,298 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 066a5683a8664e2887aa63b15655dfb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732148788675 2024-11-21T00:26:31,299 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4065282fffb647c8aaddfb5d6a0c92b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732148788731 2024-11-21T00:26:31,320 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#18 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:31,321 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/0d13046e15e74b34a88685ff59a5e046 is 50, key is test_row_0/A:col10/1732148789970/Put/seqid=0 2024-11-21T00:26:31,347 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:31,348 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/371f946d733e440988253c660dba4a14 is 50, key is test_row_0/B:col10/1732148789970/Put/seqid=0 2024-11-21T00:26:31,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741855_1031 (size=12207) 2024-11-21T00:26:31,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741856_1032 (size=12207) 2024-11-21T00:26:31,398 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/0d13046e15e74b34a88685ff59a5e046 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/0d13046e15e74b34a88685ff59a5e046 2024-11-21T00:26:31,418 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/371f946d733e440988253c660dba4a14 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/371f946d733e440988253c660dba4a14 2024-11-21T00:26:31,427 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into 0d13046e15e74b34a88685ff59a5e046(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:31,427 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:31,427 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=13, startTime=1732148791288; duration=0sec 2024-11-21T00:26:31,427 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:31,428 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:31,428 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:31,434 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:31,434 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:31,435 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,435 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/6bd57be4522f4c08bda918cda990c9a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/e46f28e57000460d8ab8dfd2c6d4052a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/2d4483b301b045de9df3eee3d4930f75] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.3 K 2024-11-21T00:26:31,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T00:26:31,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd57be4522f4c08bda918cda990c9a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148786971 2024-11-21T00:26:31,436 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:26:31,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46f28e57000460d8ab8dfd2c6d4052a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732148788675 2024-11-21T00:26:31,445 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d4483b301b045de9df3eee3d4930f75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732148788731 2024-11-21T00:26:31,448 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into 371f946d733e440988253c660dba4a14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:31,448 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:31,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/49723f20877f4210a82ab7447ab62c64 is 50, key is test_row_0/A:col10/1732148790244/Put/seqid=0 2024-11-21T00:26:31,456 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=13, startTime=1732148791290; duration=0sec 2024-11-21T00:26:31,458 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:31,458 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:31,508 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:31,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741857_1033 (size=12001) 2024-11-21T00:26:31,514 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/47ad3b2ba0a94be3bb9c3805f31d1610 is 50, key is test_row_0/C:col10/1732148789970/Put/seqid=0 2024-11-21T00:26:31,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148851504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,533 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/49723f20877f4210a82ab7447ab62c64 2024-11-21T00:26:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148851508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148851509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148851513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148851529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/fdfd580bfad5464abc4ef19806f0553b is 50, key is test_row_0/B:col10/1732148790244/Put/seqid=0 2024-11-21T00:26:31,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741858_1034 (size=12207) 2024-11-21T00:26:31,628 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/47ad3b2ba0a94be3bb9c3805f31d1610 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/47ad3b2ba0a94be3bb9c3805f31d1610 2024-11-21T00:26:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741859_1035 (size=12001) 2024-11-21T00:26:31,641 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/fdfd580bfad5464abc4ef19806f0553b 2024-11-21T00:26:31,647 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into 47ad3b2ba0a94be3bb9c3805f31d1610(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:31,647 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:31,647 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=13, startTime=1732148791292; duration=0sec 2024-11-21T00:26:31,648 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:31,648 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148851643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/4cdbe562b5784b569e68392899bf3363 is 50, key is test_row_0/C:col10/1732148790244/Put/seqid=0 2024-11-21T00:26:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148851653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148851653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148851654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148851656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:31,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741860_1036 (size=12001) 2024-11-21T00:26:31,700 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/4cdbe562b5784b569e68392899bf3363 2024-11-21T00:26:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/49723f20877f4210a82ab7447ab62c64 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/49723f20877f4210a82ab7447ab62c64 2024-11-21T00:26:31,724 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/49723f20877f4210a82ab7447ab62c64, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T00:26:31,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/fdfd580bfad5464abc4ef19806f0553b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fdfd580bfad5464abc4ef19806f0553b 2024-11-21T00:26:31,742 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fdfd580bfad5464abc4ef19806f0553b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T00:26:31,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/4cdbe562b5784b569e68392899bf3363 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4cdbe562b5784b569e68392899bf3363 2024-11-21T00:26:31,772 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4cdbe562b5784b569e68392899bf3363, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T00:26:31,774 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for b03d35ead3c53e10e140da78ec4bd637 in 338ms, sequenceid=117, compaction requested=false 2024-11-21T00:26:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T00:26:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T00:26:31,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T00:26:31,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-11-21T00:26:31,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8420 sec 2024-11-21T00:26:31,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:31,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T00:26:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:31,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/b339b91e9b1e48ffae1b34a01f35870a is 50, key is test_row_0/A:col10/1732148791863/Put/seqid=0 2024-11-21T00:26:31,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741861_1037 (size=14491) 2024-11-21T00:26:31,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/b339b91e9b1e48ffae1b34a01f35870a 2024-11-21T00:26:31,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/8371351a2cb44bc89ca732c4a34573e1 is 50, key is test_row_0/B:col10/1732148791863/Put/seqid=0 2024-11-21T00:26:32,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741862_1038 (size=12101) 2024-11-21T00:26:32,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/8371351a2cb44bc89ca732c4a34573e1 2024-11-21T00:26:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148852027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/4aaae6f9748a41c0a1369b19ed196e86 is 50, key is test_row_0/C:col10/1732148791863/Put/seqid=0 2024-11-21T00:26:32,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148852043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148852045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148852047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148852051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T00:26:32,103 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:32,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:32,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741863_1039 (size=12101) 2024-11-21T00:26:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T00:26:32,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/4aaae6f9748a41c0a1369b19ed196e86 2024-11-21T00:26:32,111 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:32,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:32,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:26:32,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/b339b91e9b1e48ffae1b34a01f35870a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/b339b91e9b1e48ffae1b34a01f35870a 2024-11-21T00:26:32,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/b339b91e9b1e48ffae1b34a01f35870a, entries=200, sequenceid=135, filesize=14.2 K 2024-11-21T00:26:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/8371351a2cb44bc89ca732c4a34573e1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8371351a2cb44bc89ca732c4a34573e1 2024-11-21T00:26:32,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8371351a2cb44bc89ca732c4a34573e1, entries=150, sequenceid=135, filesize=11.8 K 2024-11-21T00:26:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/4aaae6f9748a41c0a1369b19ed196e86 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4aaae6f9748a41c0a1369b19ed196e86 2024-11-21T00:26:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148852153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4aaae6f9748a41c0a1369b19ed196e86, entries=150, sequenceid=135, filesize=11.8 K 2024-11-21T00:26:32,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for b03d35ead3c53e10e140da78ec4bd637 in 325ms, sequenceid=135, compaction requested=true 2024-11-21T00:26:32,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:32,190 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:32,194 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:32,194 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:32,195 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,195 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/0d13046e15e74b34a88685ff59a5e046, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/49723f20877f4210a82ab7447ab62c64, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/b339b91e9b1e48ffae1b34a01f35870a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=37.8 K 2024-11-21T00:26:32,198 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d13046e15e74b34a88685ff59a5e046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732148788731 2024-11-21T00:26:32,200 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49723f20877f4210a82ab7447ab62c64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732148790173 2024-11-21T00:26:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:32,201 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b339b91e9b1e48ffae1b34a01f35870a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732148791480 2024-11-21T00:26:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:26:32,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:32,225 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:32,227 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:32,228 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:32,228 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,228 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/371f946d733e440988253c660dba4a14, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fdfd580bfad5464abc4ef19806f0553b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8371351a2cb44bc89ca732c4a34573e1] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.5 K 2024-11-21T00:26:32,230 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 371f946d733e440988253c660dba4a14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732148788731 2024-11-21T00:26:32,231 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting fdfd580bfad5464abc4ef19806f0553b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732148790173 2024-11-21T00:26:32,233 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8371351a2cb44bc89ca732c4a34573e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732148791480 2024-11-21T00:26:32,242 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:32,243 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/9573d22b2d544561969d3c7df6de5546 is 50, key is test_row_0/A:col10/1732148791863/Put/seqid=0 2024-11-21T00:26:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:32,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-21T00:26:32,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148852251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148852255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148852257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148852261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,289 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:32,289 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/7e5dffed60d94abbac4d98e331da8e0d is 50, key is test_row_0/B:col10/1732148791863/Put/seqid=0 2024-11-21T00:26:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741864_1040 (size=12409) 2024-11-21T00:26:32,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e5e70c6295644ac888c4b75253fbb0d0 is 50, key is test_row_0/A:col10/1732148792251/Put/seqid=0 2024-11-21T00:26:32,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741865_1041 (size=12409) 2024-11-21T00:26:32,375 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/7e5dffed60d94abbac4d98e331da8e0d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7e5dffed60d94abbac4d98e331da8e0d 2024-11-21T00:26:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148852373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741866_1042 (size=21715) 2024-11-21T00:26:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148852373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,391 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into 7e5dffed60d94abbac4d98e331da8e0d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:32,391 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:32,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,391 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=13, startTime=1732148792224; duration=0sec 2024-11-21T00:26:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148852383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,392 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:32,392 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:32,392 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,396 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:32,396 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:32,396 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,396 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/47ad3b2ba0a94be3bb9c3805f31d1610, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4cdbe562b5784b569e68392899bf3363, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4aaae6f9748a41c0a1369b19ed196e86] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.5 K 2024-11-21T00:26:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148852393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148852393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,403 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 47ad3b2ba0a94be3bb9c3805f31d1610, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732148788731 2024-11-21T00:26:32,404 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cdbe562b5784b569e68392899bf3363, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732148790173 2024-11-21T00:26:32,405 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aaae6f9748a41c0a1369b19ed196e86, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732148791480 2024-11-21T00:26:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:26:32,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:32,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,446 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#30 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:32,447 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8ce6fa33a1df4b97900a2c232d2f5621 is 50, key is test_row_0/C:col10/1732148791863/Put/seqid=0 2024-11-21T00:26:32,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741867_1043 (size=12409) 2024-11-21T00:26:32,518 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8ce6fa33a1df4b97900a2c232d2f5621 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ce6fa33a1df4b97900a2c232d2f5621 2024-11-21T00:26:32,532 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into 8ce6fa33a1df4b97900a2c232d2f5621(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:32,532 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:32,532 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=13, startTime=1732148792258; duration=0sec 2024-11-21T00:26:32,532 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:32,532 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:32,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148852593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:32,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148852608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148852613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148852613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148852703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,739 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/9573d22b2d544561969d3c7df6de5546 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/9573d22b2d544561969d3c7df6de5546 2024-11-21T00:26:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:26:32,758 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into 9573d22b2d544561969d3c7df6de5546(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:32,758 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:32,758 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=13, startTime=1732148792190; duration=0sec 2024-11-21T00:26:32,758 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:32,758 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:32,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:32,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:32,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e5e70c6295644ac888c4b75253fbb0d0 2024-11-21T00:26:32,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3c52e30664ec48c7877a2ca39a5dffa0 is 50, key is test_row_0/B:col10/1732148792251/Put/seqid=0 2024-11-21T00:26:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741868_1044 (size=12151) 2024-11-21T00:26:32,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148852905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148852922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148852932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148852933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:32,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:32,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:33,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148853213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:26:33,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3c52e30664ec48c7877a2ca39a5dffa0 2024-11-21T00:26:33,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/530e3608d00a4e698d490eb6fa263209 is 50, key is test_row_0/C:col10/1732148792251/Put/seqid=0 2024-11-21T00:26:33,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741869_1045 (size=12151) 2024-11-21T00:26:33,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148853421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:33,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148853432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:33,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148853443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148853443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:33,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:33,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:33,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/530e3608d00a4e698d490eb6fa263209 2024-11-21T00:26:33,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:33,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:33,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:33,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e5e70c6295644ac888c4b75253fbb0d0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e5e70c6295644ac888c4b75253fbb0d0 2024-11-21T00:26:33,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e5e70c6295644ac888c4b75253fbb0d0, entries=350, sequenceid=168, filesize=21.2 K 2024-11-21T00:26:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3c52e30664ec48c7877a2ca39a5dffa0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3c52e30664ec48c7877a2ca39a5dffa0 2024-11-21T00:26:33,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3c52e30664ec48c7877a2ca39a5dffa0, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T00:26:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/530e3608d00a4e698d490eb6fa263209 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/530e3608d00a4e698d490eb6fa263209 2024-11-21T00:26:33,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/530e3608d00a4e698d490eb6fa263209, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T00:26:33,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for b03d35ead3c53e10e140da78ec4bd637 in 1655ms, sequenceid=168, compaction requested=false 2024-11-21T00:26:33,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:33,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T00:26:33,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:33,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:33,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T00:26:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T00:26:33,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T00:26:33,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-11-21T00:26:33,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8430 sec 2024-11-21T00:26:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T00:26:34,273 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:34,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T00:26:34,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:26:34,279 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:34,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:26:34,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:34,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:26:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:34,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/51968439db5943caa379b5ae61549a42 is 50, key is test_row_0/A:col10/1732148794396/Put/seqid=0 2024-11-21T00:26:34,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:26:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741870_1046 (size=14541) 2024-11-21T00:26:34,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/51968439db5943caa379b5ae61549a42 2024-11-21T00:26:34,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/b35d20680ce643278c00c3e705947ddc is 50, key is test_row_0/B:col10/1732148794396/Put/seqid=0 2024-11-21T00:26:34,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:26:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:26:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741871_1047 (size=12151) 2024-11-21T00:26:34,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/b35d20680ce643278c00c3e705947ddc 2024-11-21T00:26:34,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3b94b11f3e1b428ba768d9adaa91e991 is 50, key is test_row_0/C:col10/1732148794396/Put/seqid=0 2024-11-21T00:26:34,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148854631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148854635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148854641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148854686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148854693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741872_1048 (size=12151) 2024-11-21T00:26:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3b94b11f3e1b428ba768d9adaa91e991 2024-11-21T00:26:34,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/51968439db5943caa379b5ae61549a42 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/51968439db5943caa379b5ae61549a42 2024-11-21T00:26:34,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/51968439db5943caa379b5ae61549a42, entries=200, sequenceid=183, filesize=14.2 K 2024-11-21T00:26:34,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/b35d20680ce643278c00c3e705947ddc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/b35d20680ce643278c00c3e705947ddc 2024-11-21T00:26:34,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:26:34,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:34,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148854793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148854794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148854794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/b35d20680ce643278c00c3e705947ddc, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T00:26:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3b94b11f3e1b428ba768d9adaa91e991 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3b94b11f3e1b428ba768d9adaa91e991 2024-11-21T00:26:34,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148854815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148854816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:34,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3b94b11f3e1b428ba768d9adaa91e991, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T00:26:34,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b03d35ead3c53e10e140da78ec4bd637 in 449ms, sequenceid=183, compaction requested=true 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:34,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T00:26:34,849 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:34,850 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:34,852 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:34,852 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:34,852 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 48665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:34,852 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,852 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:34,852 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,852 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ce6fa33a1df4b97900a2c232d2f5621, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/530e3608d00a4e698d490eb6fa263209, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3b94b11f3e1b428ba768d9adaa91e991] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.9 K 2024-11-21T00:26:34,852 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/9573d22b2d544561969d3c7df6de5546, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e5e70c6295644ac888c4b75253fbb0d0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/51968439db5943caa379b5ae61549a42] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=47.5 K 2024-11-21T00:26:34,853 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce6fa33a1df4b97900a2c232d2f5621, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732148791480 2024-11-21T00:26:34,854 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9573d22b2d544561969d3c7df6de5546, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732148791480 2024-11-21T00:26:34,854 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 530e3608d00a4e698d490eb6fa263209, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732148792217 2024-11-21T00:26:34,854 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5e70c6295644ac888c4b75253fbb0d0, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732148792217 2024-11-21T00:26:34,855 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b94b11f3e1b428ba768d9adaa91e991, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732148794284 2024-11-21T00:26:34,856 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51968439db5943caa379b5ae61549a42, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732148794284 2024-11-21T00:26:34,881 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#36 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:34,882 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/bb669373b41d48379c0a2b00a8d97fde is 50, key is test_row_0/C:col10/1732148794396/Put/seqid=0 2024-11-21T00:26:34,899 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:34,900 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/f20a04728abb4baa994d4329038fb142 is 50, key is test_row_0/A:col10/1732148794396/Put/seqid=0 2024-11-21T00:26:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:26:34,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T00:26:34,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,934 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:26:34,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741873_1049 (size=12561) 2024-11-21T00:26:34,958 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/bb669373b41d48379c0a2b00a8d97fde as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/bb669373b41d48379c0a2b00a8d97fde 2024-11-21T00:26:34,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741874_1050 (size=12561) 2024-11-21T00:26:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/8a092521c40e43ae9fe2fc1961a73250 is 50, key is test_row_0/A:col10/1732148794624/Put/seqid=0 2024-11-21T00:26:34,992 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into bb669373b41d48379c0a2b00a8d97fde(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:34,992 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:34,992 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=13, startTime=1732148794849; duration=0sec 2024-11-21T00:26:34,992 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:34,993 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:34,993 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:34,996 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:34,997 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:34,997 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:34,997 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7e5dffed60d94abbac4d98e331da8e0d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3c52e30664ec48c7877a2ca39a5dffa0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/b35d20680ce643278c00c3e705947ddc] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=35.9 K 2024-11-21T00:26:34,998 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5dffed60d94abbac4d98e331da8e0d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732148791480 2024-11-21T00:26:35,004 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c52e30664ec48c7877a2ca39a5dffa0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732148792217 2024-11-21T00:26:35,005 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b35d20680ce643278c00c3e705947ddc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732148794284 2024-11-21T00:26:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741875_1051 (size=12151) 2024-11-21T00:26:35,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:35,024 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/8a092521c40e43ae9fe2fc1961a73250 2024-11-21T00:26:35,047 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:35,048 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3ab1db70f3eb4747a89e91fce1882ebc is 50, key is test_row_0/B:col10/1732148794396/Put/seqid=0 2024-11-21T00:26:35,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/46a7d1baa5ad4db9a5f5ffbe67d23b87 is 50, key is test_row_0/B:col10/1732148794624/Put/seqid=0 2024-11-21T00:26:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148855057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148855057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148855065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148855069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148855070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741876_1052 (size=12561) 2024-11-21T00:26:35,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741877_1053 (size=12151) 2024-11-21T00:26:35,158 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3ab1db70f3eb4747a89e91fce1882ebc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3ab1db70f3eb4747a89e91fce1882ebc 2024-11-21T00:26:35,163 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/46a7d1baa5ad4db9a5f5ffbe67d23b87 2024-11-21T00:26:35,174 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into 3ab1db70f3eb4747a89e91fce1882ebc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:35,174 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:35,174 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=13, startTime=1732148794849; duration=0sec 2024-11-21T00:26:35,174 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:35,174 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148855192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148855193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/5ba8e2ee04dd4c8c8660bfe6293f312d is 50, key is test_row_0/C:col10/1732148794624/Put/seqid=0 2024-11-21T00:26:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148855203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741878_1054 (size=12151) 2024-11-21T00:26:35,257 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/5ba8e2ee04dd4c8c8660bfe6293f312d 2024-11-21T00:26:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/8a092521c40e43ae9fe2fc1961a73250 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/8a092521c40e43ae9fe2fc1961a73250 2024-11-21T00:26:35,284 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/8a092521c40e43ae9fe2fc1961a73250, entries=150, sequenceid=209, filesize=11.9 K 2024-11-21T00:26:35,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/46a7d1baa5ad4db9a5f5ffbe67d23b87 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/46a7d1baa5ad4db9a5f5ffbe67d23b87 2024-11-21T00:26:35,299 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/46a7d1baa5ad4db9a5f5ffbe67d23b87, entries=150, sequenceid=209, filesize=11.9 K 2024-11-21T00:26:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/5ba8e2ee04dd4c8c8660bfe6293f312d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5ba8e2ee04dd4c8c8660bfe6293f312d 2024-11-21T00:26:35,321 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5ba8e2ee04dd4c8c8660bfe6293f312d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-21T00:26:35,325 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for b03d35ead3c53e10e140da78ec4bd637 in 391ms, sequenceid=209, compaction requested=false 2024-11-21T00:26:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T00:26:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T00:26:35,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T00:26:35,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0500 sec 2024-11-21T00:26:35,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0630 sec 2024-11-21T00:26:35,386 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/f20a04728abb4baa994d4329038fb142 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f20a04728abb4baa994d4329038fb142 2024-11-21T00:26:35,391 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:35,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T00:26:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:35,403 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into f20a04728abb4baa994d4329038fb142(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:35,403 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:35,403 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=13, startTime=1732148794849; duration=0sec 2024-11-21T00:26:35,403 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:35,404 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:35,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/84049daad1084eaeafb46c838c8863b9 is 50, key is test_row_0/A:col10/1732148795065/Put/seqid=0 2024-11-21T00:26:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T00:26:35,423 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:35,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T00:26:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:35,429 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:35,431 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:35,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:35,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741879_1055 (size=14541) 2024-11-21T00:26:35,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/84049daad1084eaeafb46c838c8863b9 2024-11-21T00:26:35,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/db1b46e295ff48d898ea37d854b0812c is 50, key is test_row_0/B:col10/1732148795065/Put/seqid=0 2024-11-21T00:26:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148855468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148855476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148855478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148855483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148855486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741880_1056 (size=12151) 2024-11-21T00:26:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:35,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:35,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:35,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148855593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148855593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148855603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148855603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148855613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148855813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148855814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148855822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148855829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148855841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:35,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:35,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/db1b46e295ff48d898ea37d854b0812c 2024-11-21T00:26:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:35,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d6027366748641d596353526e8b8980e is 50, key is test_row_0/C:col10/1732148795065/Put/seqid=0 2024-11-21T00:26:36,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741881_1057 (size=12151) 2024-11-21T00:26:36,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d6027366748641d596353526e8b8980e 2024-11-21T00:26:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:36,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/84049daad1084eaeafb46c838c8863b9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/84049daad1084eaeafb46c838c8863b9 2024-11-21T00:26:36,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/84049daad1084eaeafb46c838c8863b9, entries=200, sequenceid=223, filesize=14.2 K 2024-11-21T00:26:36,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/db1b46e295ff48d898ea37d854b0812c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/db1b46e295ff48d898ea37d854b0812c 2024-11-21T00:26:36,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/db1b46e295ff48d898ea37d854b0812c, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T00:26:36,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d6027366748641d596353526e8b8980e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d6027366748641d596353526e8b8980e 2024-11-21T00:26:36,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148856123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148856124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d6027366748641d596353526e8b8980e, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T00:26:36,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b03d35ead3c53e10e140da78ec4bd637 in 760ms, sequenceid=223, compaction requested=true 2024-11-21T00:26:36,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:36,152 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:36,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:36,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:36,154 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:36,155 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:36,155 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:36,155 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,155 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f20a04728abb4baa994d4329038fb142, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/8a092521c40e43ae9fe2fc1961a73250, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/84049daad1084eaeafb46c838c8863b9] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=38.3 K 2024-11-21T00:26:36,155 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:36,156 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:36,156 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,156 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3ab1db70f3eb4747a89e91fce1882ebc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/46a7d1baa5ad4db9a5f5ffbe67d23b87, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/db1b46e295ff48d898ea37d854b0812c] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=36.0 K 2024-11-21T00:26:36,157 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f20a04728abb4baa994d4329038fb142, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732148794284 2024-11-21T00:26:36,157 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab1db70f3eb4747a89e91fce1882ebc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732148794284 2024-11-21T00:26:36,158 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a092521c40e43ae9fe2fc1961a73250, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732148794624 2024-11-21T00:26:36,158 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a7d1baa5ad4db9a5f5ffbe67d23b87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732148794624 2024-11-21T00:26:36,160 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84049daad1084eaeafb46c838c8863b9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148795042 2024-11-21T00:26:36,160 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting db1b46e295ff48d898ea37d854b0812c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148795044 2024-11-21T00:26:36,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:36,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:36,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:36,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:36,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:36,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:26:36,230 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:36,231 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/ab4dda363a6b41d0a7f51bf434ccfd0c is 50, key is test_row_0/A:col10/1732148795065/Put/seqid=0 2024-11-21T00:26:36,240 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:36,241 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/04209edc23944e2a89d0c84ae9629f13 is 50, key is test_row_0/B:col10/1732148795065/Put/seqid=0 2024-11-21T00:26:36,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:36,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/5533a9db3cc84d698b549c8711428aa6 is 50, key is test_row_0/A:col10/1732148795468/Put/seqid=0 2024-11-21T00:26:36,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148856255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:36,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148856275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148856278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741882_1058 (size=12663) 2024-11-21T00:26:36,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741883_1059 (size=12663) 2024-11-21T00:26:36,363 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/04209edc23944e2a89d0c84ae9629f13 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/04209edc23944e2a89d0c84ae9629f13 2024-11-21T00:26:36,377 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into 04209edc23944e2a89d0c84ae9629f13(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:36,377 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:36,377 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=13, startTime=1732148796154; duration=0sec 2024-11-21T00:26:36,379 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:36,379 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:36,379 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:36,381 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:36,381 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:36,381 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,381 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/bb669373b41d48379c0a2b00a8d97fde, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5ba8e2ee04dd4c8c8660bfe6293f312d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d6027366748641d596353526e8b8980e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=36.0 K 2024-11-21T00:26:36,382 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bb669373b41d48379c0a2b00a8d97fde, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732148794284 2024-11-21T00:26:36,383 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba8e2ee04dd4c8c8660bfe6293f312d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732148794624 2024-11-21T00:26:36,384 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d6027366748641d596353526e8b8980e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148795044 2024-11-21T00:26:36,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148856382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741884_1060 (size=12151) 2024-11-21T00:26:36,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/5533a9db3cc84d698b549c8711428aa6 2024-11-21T00:26:36,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148856393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,411 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:36,412 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/519366c35aa442a4bd545616d03559ee is 50, key is test_row_0/C:col10/1732148795065/Put/seqid=0 2024-11-21T00:26:36,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148856403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/beb9f220e4134b05bf7e14d41b9ee854 is 50, key is test_row_0/B:col10/1732148795468/Put/seqid=0 2024-11-21T00:26:36,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741885_1061 (size=12663) 2024-11-21T00:26:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741886_1062 (size=12151) 2024-11-21T00:26:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:36,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:36,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148856605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148856621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148856622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148856654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148856656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,747 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/ab4dda363a6b41d0a7f51bf434ccfd0c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/ab4dda363a6b41d0a7f51bf434ccfd0c 2024-11-21T00:26:36,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:36,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,760 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into ab4dda363a6b41d0a7f51bf434ccfd0c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:36,760 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:36,760 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=13, startTime=1732148796152; duration=0sec 2024-11-21T00:26:36,760 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:36,760 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:36,913 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/519366c35aa442a4bd545616d03559ee as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/519366c35aa442a4bd545616d03559ee 2024-11-21T00:26:36,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:36,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/beb9f220e4134b05bf7e14d41b9ee854 2024-11-21T00:26:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:36,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:36,929 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into 519366c35aa442a4bd545616d03559ee(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:36,929 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:36,929 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=13, startTime=1732148796169; duration=0sec 2024-11-21T00:26:36,929 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:36,930 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:36,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148856924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148856952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:36,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148856953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:36,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/892337f436e747ed96adf9bb77fb96e4 is 50, key is test_row_0/C:col10/1732148795468/Put/seqid=0 2024-11-21T00:26:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741887_1063 (size=12151) 2024-11-21T00:26:37,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/892337f436e747ed96adf9bb77fb96e4 2024-11-21T00:26:37,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/5533a9db3cc84d698b549c8711428aa6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/5533a9db3cc84d698b549c8711428aa6 2024-11-21T00:26:37,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/5533a9db3cc84d698b549c8711428aa6, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T00:26:37,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/beb9f220e4134b05bf7e14d41b9ee854 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/beb9f220e4134b05bf7e14d41b9ee854 2024-11-21T00:26:37,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/beb9f220e4134b05bf7e14d41b9ee854, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T00:26:37,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/892337f436e747ed96adf9bb77fb96e4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/892337f436e747ed96adf9bb77fb96e4 2024-11-21T00:26:37,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/892337f436e747ed96adf9bb77fb96e4, entries=150, sequenceid=251, filesize=11.9 K 2024-11-21T00:26:37,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for b03d35ead3c53e10e140da78ec4bd637 in 871ms, sequenceid=251, compaction requested=false 2024-11-21T00:26:37,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:37,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T00:26:37,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:37,082 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T00:26:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e44ec91b9ac844d6af88494516e5bf52 is 50, key is test_row_0/A:col10/1732148796259/Put/seqid=0 2024-11-21T00:26:37,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741888_1064 (size=12301) 2024-11-21T00:26:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:37,530 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e44ec91b9ac844d6af88494516e5bf52 2024-11-21T00:26:37,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3623651ed14d4541bef0f851aadfeec9 is 50, key is test_row_0/B:col10/1732148796259/Put/seqid=0 2024-11-21T00:26:37,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741889_1065 (size=12301) 2024-11-21T00:26:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:37,573 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3623651ed14d4541bef0f851aadfeec9 2024-11-21T00:26:37,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c3b85ed10d6749ddbea1f08a360090a0 is 50, key is test_row_0/C:col10/1732148796259/Put/seqid=0 2024-11-21T00:26:37,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741890_1066 (size=12301) 2024-11-21T00:26:37,655 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c3b85ed10d6749ddbea1f08a360090a0 2024-11-21T00:26:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e44ec91b9ac844d6af88494516e5bf52 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e44ec91b9ac844d6af88494516e5bf52 2024-11-21T00:26:37,686 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e44ec91b9ac844d6af88494516e5bf52, entries=150, sequenceid=264, filesize=12.0 K 2024-11-21T00:26:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3623651ed14d4541bef0f851aadfeec9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3623651ed14d4541bef0f851aadfeec9 2024-11-21T00:26:37,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148857683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148857688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,708 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,714 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3623651ed14d4541bef0f851aadfeec9, entries=150, sequenceid=264, filesize=12.0 K 2024-11-21T00:26:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c3b85ed10d6749ddbea1f08a360090a0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3b85ed10d6749ddbea1f08a360090a0 2024-11-21T00:26:37,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148857703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148857704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,728 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3b85ed10d6749ddbea1f08a360090a0, entries=150, sequenceid=264, filesize=12.0 K 2024-11-21T00:26:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,730 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for b03d35ead3c53e10e140da78ec4bd637 in 648ms, sequenceid=264, compaction requested=true 2024-11-21T00:26:37,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:37,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:37,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T00:26:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T00:26:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:37,733 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:37,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T00:26:37,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T00:26:37,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3020 sec 2024-11-21T00:26:37,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.3250 sec 2024-11-21T00:26:37,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/716c93bb7a024f26b7c314e9f1401fb6 is 50, key is test_row_0/A:col10/1732148797698/Put/seqid=0 2024-11-21T00:26:37,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741891_1067 (size=17181) 2024-11-21T00:26:37,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/716c93bb7a024f26b7c314e9f1401fb6 2024-11-21T00:26:37,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148857801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148857813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148857832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148857832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:37,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/bc8a6b65521240dc97ae9517e0fb3297 is 50, key is test_row_0/B:col10/1732148797698/Put/seqid=0 2024-11-21T00:26:37,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741892_1068 (size=12301) 2024-11-21T00:26:37,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148857922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148858022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148858062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148858062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148858142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/bc8a6b65521240dc97ae9517e0fb3297 2024-11-21T00:26:38,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/1df42d42a25a49a4936e3f87321e5a42 is 50, key is test_row_0/C:col10/1732148797698/Put/seqid=0 2024-11-21T00:26:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148858332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741893_1069 (size=12301) 2024-11-21T00:26:38,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/1df42d42a25a49a4936e3f87321e5a42 2024-11-21T00:26:38,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/716c93bb7a024f26b7c314e9f1401fb6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/716c93bb7a024f26b7c314e9f1401fb6 2024-11-21T00:26:38,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/716c93bb7a024f26b7c314e9f1401fb6, entries=250, sequenceid=292, filesize=16.8 K 2024-11-21T00:26:38,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/bc8a6b65521240dc97ae9517e0fb3297 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/bc8a6b65521240dc97ae9517e0fb3297 2024-11-21T00:26:38,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/bc8a6b65521240dc97ae9517e0fb3297, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T00:26:38,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/1df42d42a25a49a4936e3f87321e5a42 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/1df42d42a25a49a4936e3f87321e5a42 2024-11-21T00:26:38,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/1df42d42a25a49a4936e3f87321e5a42, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T00:26:38,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148858373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b03d35ead3c53e10e140da78ec4bd637 in 654ms, sequenceid=292, compaction requested=true 2024-11-21T00:26:38,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:38,388 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:26:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:38,389 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:26:38,390 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54296 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:26:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:38,390 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:38,391 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:38,391 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/ab4dda363a6b41d0a7f51bf434ccfd0c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/5533a9db3cc84d698b549c8711428aa6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e44ec91b9ac844d6af88494516e5bf52, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/716c93bb7a024f26b7c314e9f1401fb6] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=53.0 K 2024-11-21T00:26:38,391 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab4dda363a6b41d0a7f51bf434ccfd0c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148795044 2024-11-21T00:26:38,392 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:26:38,392 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:38,392 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:38,392 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/04209edc23944e2a89d0c84ae9629f13, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/beb9f220e4134b05bf7e14d41b9ee854, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3623651ed14d4541bef0f851aadfeec9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/bc8a6b65521240dc97ae9517e0fb3297] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=48.3 K 2024-11-21T00:26:38,392 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5533a9db3cc84d698b549c8711428aa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732148795468 2024-11-21T00:26:38,393 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 04209edc23944e2a89d0c84ae9629f13, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148795044 2024-11-21T00:26:38,393 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44ec91b9ac844d6af88494516e5bf52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732148796249 2024-11-21T00:26:38,393 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting beb9f220e4134b05bf7e14d41b9ee854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732148795468 2024-11-21T00:26:38,394 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3623651ed14d4541bef0f851aadfeec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732148796249 2024-11-21T00:26:38,394 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716c93bb7a024f26b7c314e9f1401fb6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148797672 2024-11-21T00:26:38,394 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8a6b65521240dc97ae9517e0fb3297, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148797698 2024-11-21T00:26:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:38,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:38,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:26:38,425 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#57 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:38,426 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/fd3e2ea8941c4f5fb097739bd75c0869 is 50, key is test_row_0/B:col10/1732148797698/Put/seqid=0 2024-11-21T00:26:38,429 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#58 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:38,431 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/45e0594d220441cb8757f7fee4f21754 is 50, key is test_row_0/A:col10/1732148797698/Put/seqid=0 2024-11-21T00:26:38,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/770627a2dd38493a8daa84515a382003 is 50, key is test_row_0/A:col10/1732148798386/Put/seqid=0 2024-11-21T00:26:38,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741894_1070 (size=12949) 2024-11-21T00:26:38,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741895_1071 (size=12949) 2024-11-21T00:26:38,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741896_1072 (size=19621) 2024-11-21T00:26:38,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/770627a2dd38493a8daa84515a382003 2024-11-21T00:26:38,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/0e55bf62ccb6439fb57150914b172c0f is 50, key is test_row_0/B:col10/1732148798386/Put/seqid=0 2024-11-21T00:26:38,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741897_1073 (size=12301) 2024-11-21T00:26:38,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/0e55bf62ccb6439fb57150914b172c0f 2024-11-21T00:26:38,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/5be8012f913840f897eb9424b8268e37 is 50, key is test_row_0/C:col10/1732148798386/Put/seqid=0 2024-11-21T00:26:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148858648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148858648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741898_1074 (size=12301) 2024-11-21T00:26:38,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/5be8012f913840f897eb9424b8268e37 2024-11-21T00:26:38,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/770627a2dd38493a8daa84515a382003 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/770627a2dd38493a8daa84515a382003 2024-11-21T00:26:38,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/770627a2dd38493a8daa84515a382003, entries=300, sequenceid=303, filesize=19.2 K 2024-11-21T00:26:38,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/0e55bf62ccb6439fb57150914b172c0f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/0e55bf62ccb6439fb57150914b172c0f 2024-11-21T00:26:38,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/0e55bf62ccb6439fb57150914b172c0f, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T00:26:38,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/5be8012f913840f897eb9424b8268e37 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5be8012f913840f897eb9424b8268e37 2024-11-21T00:26:38,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5be8012f913840f897eb9424b8268e37, entries=150, sequenceid=303, filesize=12.0 K 2024-11-21T00:26:38,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b03d35ead3c53e10e140da78ec4bd637 in 334ms, sequenceid=303, compaction requested=true 2024-11-21T00:26:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:38,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:26:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T00:26:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-21T00:26:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:38,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:38,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:26:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148858791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148858795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/32b6d46e3e654eeaa1580c6252a6abff is 50, key is test_row_0/A:col10/1732148798629/Put/seqid=0 2024-11-21T00:26:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148858842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741899_1075 (size=14741) 2024-11-21T00:26:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148858894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148858903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,913 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/fd3e2ea8941c4f5fb097739bd75c0869 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fd3e2ea8941c4f5fb097739bd75c0869 2024-11-21T00:26:38,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148858912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:38,928 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/45e0594d220441cb8757f7fee4f21754 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/45e0594d220441cb8757f7fee4f21754 2024-11-21T00:26:38,937 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into fd3e2ea8941c4f5fb097739bd75c0869(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:38,937 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:38,937 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=12, startTime=1732148798389; duration=0sec 2024-11-21T00:26:38,937 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-21T00:26:38,937 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:38,938 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-21T00:26:38,938 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:26:38,939 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:26:38,939 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. because compaction request was cancelled 2024-11-21T00:26:38,939 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:38,939 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:26:38,940 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into 45e0594d220441cb8757f7fee4f21754(size=12.6 K), total size for store is 31.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:38,940 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:38,940 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=12, startTime=1732148798387; duration=0sec 2024-11-21T00:26:38,940 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:26:38,940 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:38,940 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:38,947 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61717 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:26:38,948 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:38,948 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-21T00:26:38,948 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:26:38,949 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:38,948 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:26:38,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. because compaction request was cancelled 2024-11-21T00:26:38,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:38,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:26:38,950 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/519366c35aa442a4bd545616d03559ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/892337f436e747ed96adf9bb77fb96e4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3b85ed10d6749ddbea1f08a360090a0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/1df42d42a25a49a4936e3f87321e5a42, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5be8012f913840f897eb9424b8268e37] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=60.3 K 2024-11-21T00:26:38,951 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 519366c35aa442a4bd545616d03559ee, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732148795044 2024-11-21T00:26:38,952 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:26:38,952 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:26:38,952 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. because compaction request was cancelled 2024-11-21T00:26:38,952 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:38,952 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 892337f436e747ed96adf9bb77fb96e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732148795468 2024-11-21T00:26:38,953 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b85ed10d6749ddbea1f08a360090a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732148796249 2024-11-21T00:26:38,953 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df42d42a25a49a4936e3f87321e5a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148797698 2024-11-21T00:26:38,954 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be8012f913840f897eb9424b8268e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148797781 2024-11-21T00:26:39,027 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:39,029 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8c91c19d54114e7eaea89f0c8ecde945 is 50, key is test_row_0/C:col10/1732148798386/Put/seqid=0 2024-11-21T00:26:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741900_1076 (size=12983) 2024-11-21T00:26:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148859112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:39,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148859133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:39,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/32b6d46e3e654eeaa1580c6252a6abff 2024-11-21T00:26:39,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/7663a66bfa1944caa4fe86fcff6e07d1 is 50, key is test_row_0/B:col10/1732148798629/Put/seqid=0 2024-11-21T00:26:39,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741901_1077 (size=12301) 2024-11-21T00:26:39,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/7663a66bfa1944caa4fe86fcff6e07d1 2024-11-21T00:26:39,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/897e0a56ac024eb88686b4558ac334de is 50, key is test_row_0/C:col10/1732148798629/Put/seqid=0 2024-11-21T00:26:39,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148859425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:39,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741902_1078 (size=12301) 2024-11-21T00:26:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148859447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:39,509 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8c91c19d54114e7eaea89f0c8ecde945 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8c91c19d54114e7eaea89f0c8ecde945 2024-11-21T00:26:39,532 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into 8c91c19d54114e7eaea89f0c8ecde945(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:39,532 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:39,533 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=11, startTime=1732148798746; duration=0sec 2024-11-21T00:26:39,533 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:39,533 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T00:26:39,593 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:39,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T00:26:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:26:39,599 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:39,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:39,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:26:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148859722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:39,728 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:39,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T00:26:39,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:39,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:39,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:39,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:39,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:39,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/897e0a56ac024eb88686b4558ac334de 2024-11-21T00:26:39,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/32b6d46e3e654eeaa1580c6252a6abff as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/32b6d46e3e654eeaa1580c6252a6abff 2024-11-21T00:26:39,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/32b6d46e3e654eeaa1580c6252a6abff, entries=200, sequenceid=331, filesize=14.4 K 2024-11-21T00:26:39,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148859862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:39,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/7663a66bfa1944caa4fe86fcff6e07d1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7663a66bfa1944caa4fe86fcff6e07d1 2024-11-21T00:26:39,877 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:39,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7663a66bfa1944caa4fe86fcff6e07d1, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T00:26:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/897e0a56ac024eb88686b4558ac334de as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/897e0a56ac024eb88686b4558ac334de 2024-11-21T00:26:39,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/897e0a56ac024eb88686b4558ac334de, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T00:26:39,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b03d35ead3c53e10e140da78ec4bd637 in 1138ms, sequenceid=331, compaction requested=true 2024-11-21T00:26:39,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:39,908 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:39,910 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:39,910 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:39,910 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:39,910 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/45e0594d220441cb8757f7fee4f21754, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/770627a2dd38493a8daa84515a382003, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/32b6d46e3e654eeaa1580c6252a6abff] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=46.2 K 2024-11-21T00:26:39,911 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e0594d220441cb8757f7fee4f21754, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148797698 2024-11-21T00:26:39,911 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770627a2dd38493a8daa84515a382003, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148797771 2024-11-21T00:26:39,911 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b6d46e3e654eeaa1580c6252a6abff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148798629 2024-11-21T00:26:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:26:39,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:39,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:39,915 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:26:39,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:39,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:39,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:39,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:39,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T00:26:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:39,918 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T00:26:39,920 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:26:39,920 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:39,920 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:39,921 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fd3e2ea8941c4f5fb097739bd75c0869, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/0e55bf62ccb6439fb57150914b172c0f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7663a66bfa1944caa4fe86fcff6e07d1] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=36.7 K 2024-11-21T00:26:39,923 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3e2ea8941c4f5fb097739bd75c0869, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148797698 2024-11-21T00:26:39,926 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e55bf62ccb6439fb57150914b172c0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148797781 2024-11-21T00:26:39,927 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7663a66bfa1944caa4fe86fcff6e07d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148798629 2024-11-21T00:26:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/22d05eb4a6774f9eb411fcd4bf8a198b is 50, key is test_row_0/A:col10/1732148798788/Put/seqid=0 2024-11-21T00:26:39,956 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#67 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:39,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:39,957 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/fea5a71d6a684d02ac09a5d754f0d5fa is 50, key is test_row_0/A:col10/1732148798629/Put/seqid=0 2024-11-21T00:26:39,965 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#68 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:39,966 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/e4b3ab85b0ed445fa25900445cd6e29c is 50, key is test_row_0/B:col10/1732148798629/Put/seqid=0 2024-11-21T00:26:40,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741903_1079 (size=9857) 2024-11-21T00:26:40,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741904_1080 (size=13051) 2024-11-21T00:26:40,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741905_1081 (size=13051) 2024-11-21T00:26:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:26:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148860222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148860222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148860223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148860333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148860333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148860333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,412 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/22d05eb4a6774f9eb411fcd4bf8a198b 2024-11-21T00:26:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/29da274bf7004455b7ab7dd1abcffa7e is 50, key is test_row_0/B:col10/1732148798788/Put/seqid=0 2024-11-21T00:26:40,459 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/fea5a71d6a684d02ac09a5d754f0d5fa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/fea5a71d6a684d02ac09a5d754f0d5fa 2024-11-21T00:26:40,471 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/e4b3ab85b0ed445fa25900445cd6e29c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e4b3ab85b0ed445fa25900445cd6e29c 2024-11-21T00:26:40,480 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into e4b3ab85b0ed445fa25900445cd6e29c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:40,480 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:40,480 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=13, startTime=1732148799915; duration=0sec 2024-11-21T00:26:40,481 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:40,481 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:40,481 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:26:40,482 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:26:40,482 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:26:40,482 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. because compaction request was cancelled 2024-11-21T00:26:40,482 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:40,492 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into fea5a71d6a684d02ac09a5d754f0d5fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:40,492 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:40,492 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=13, startTime=1732148799908; duration=0sec 2024-11-21T00:26:40,493 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:40,493 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:40,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741906_1082 (size=9857) 2024-11-21T00:26:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148860542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148860543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148860544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:26:40,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148860864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148860866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148860863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:40,909 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/29da274bf7004455b7ab7dd1abcffa7e 2024-11-21T00:26:40,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/b6dba80880584786b55bab6e463f7ef3 is 50, key is test_row_0/C:col10/1732148798788/Put/seqid=0 2024-11-21T00:26:40,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741907_1083 (size=9857) 2024-11-21T00:26:41,377 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/b6dba80880584786b55bab6e463f7ef3 2024-11-21T00:26:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148861383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148861383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148861383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:41,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/22d05eb4a6774f9eb411fcd4bf8a198b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/22d05eb4a6774f9eb411fcd4bf8a198b 2024-11-21T00:26:41,403 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/22d05eb4a6774f9eb411fcd4bf8a198b, entries=100, sequenceid=342, filesize=9.6 K 2024-11-21T00:26:41,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/29da274bf7004455b7ab7dd1abcffa7e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/29da274bf7004455b7ab7dd1abcffa7e 2024-11-21T00:26:41,412 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/29da274bf7004455b7ab7dd1abcffa7e, entries=100, sequenceid=342, filesize=9.6 K 2024-11-21T00:26:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/b6dba80880584786b55bab6e463f7ef3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/b6dba80880584786b55bab6e463f7ef3 2024-11-21T00:26:41,430 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/b6dba80880584786b55bab6e463f7ef3, entries=100, sequenceid=342, filesize=9.6 K 2024-11-21T00:26:41,432 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for b03d35ead3c53e10e140da78ec4bd637 in 1513ms, sequenceid=342, compaction requested=true 2024-11-21T00:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T00:26:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T00:26:41,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T00:26:41,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-21T00:26:41,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8440 sec 2024-11-21T00:26:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T00:26:41,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:41,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T00:26:41,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:26:41,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:41,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:26:41,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T00:26:41,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:41,901 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-21T00:26:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:41,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e0d9f87782e54a80ab324fbaae606239 is 50, key is test_row_0/A:col10/1732148800173/Put/seqid=0 2024-11-21T00:26:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741908_1084 (size=12301) 2024-11-21T00:26:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148861938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148862063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:26:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148862273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,345 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e0d9f87782e54a80ab324fbaae606239 2024-11-21T00:26:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/ded20427d6fe4536b594db0e3c471f76 is 50, key is test_row_0/B:col10/1732148800173/Put/seqid=0 2024-11-21T00:26:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:26:42,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148862402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:42,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148862403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:42,405 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:42,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741909_1085 (size=12301) 2024-11-21T00:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,407 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148862403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,408 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,410 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/ded20427d6fe4536b594db0e3c471f76 2024-11-21T00:26:42,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c147f38cfa314119982a8fb13fdad87e is 50, key is test_row_0/C:col10/1732148800173/Put/seqid=0 2024-11-21T00:26:42,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741910_1086 (size=12301) 2024-11-21T00:26:42,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148862592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:42,860 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c147f38cfa314119982a8fb13fdad87e 2024-11-21T00:26:42,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e0d9f87782e54a80ab324fbaae606239 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e0d9f87782e54a80ab324fbaae606239 2024-11-21T00:26:42,883 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e0d9f87782e54a80ab324fbaae606239, entries=150, sequenceid=373, filesize=12.0 K 2024-11-21T00:26:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/ded20427d6fe4536b594db0e3c471f76 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ded20427d6fe4536b594db0e3c471f76 2024-11-21T00:26:42,898 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ded20427d6fe4536b594db0e3c471f76, entries=150, sequenceid=373, filesize=12.0 K 2024-11-21T00:26:42,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c147f38cfa314119982a8fb13fdad87e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c147f38cfa314119982a8fb13fdad87e 2024-11-21T00:26:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:26:42,914 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c147f38cfa314119982a8fb13fdad87e, entries=150, sequenceid=373, filesize=12.0 K 2024-11-21T00:26:42,916 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for b03d35ead3c53e10e140da78ec4bd637 in 1014ms, sequenceid=373, compaction requested=true 2024-11-21T00:26:42,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:42,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:42,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T00:26:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T00:26:42,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T00:26:42,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-11-21T00:26:42,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1860 sec 2024-11-21T00:26:43,137 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:43,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:26:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:43,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/dd2e153bb9d84f7cbcbd74a73a9f18db is 50, key is test_row_0/A:col10/1732148803135/Put/seqid=0 2024-11-21T00:26:43,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741911_1087 (size=14741) 2024-11-21T00:26:43,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/dd2e153bb9d84f7cbcbd74a73a9f18db 2024-11-21T00:26:43,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3a19beee01fe4c238917abfb77831938 is 50, key is test_row_0/B:col10/1732148803135/Put/seqid=0 2024-11-21T00:26:43,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741912_1088 (size=12301) 2024-11-21T00:26:43,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3a19beee01fe4c238917abfb77831938 2024-11-21T00:26:43,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d3016dc2d2544e50b5be03deb5bd1903 is 50, key is test_row_0/C:col10/1732148803135/Put/seqid=0 2024-11-21T00:26:43,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741913_1089 (size=12301) 2024-11-21T00:26:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148863362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148863482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:43,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148863702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:43,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d3016dc2d2544e50b5be03deb5bd1903 2024-11-21T00:26:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/dd2e153bb9d84f7cbcbd74a73a9f18db as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/dd2e153bb9d84f7cbcbd74a73a9f18db 2024-11-21T00:26:43,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/dd2e153bb9d84f7cbcbd74a73a9f18db, entries=200, sequenceid=384, filesize=14.4 K 2024-11-21T00:26:43,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/3a19beee01fe4c238917abfb77831938 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3a19beee01fe4c238917abfb77831938 2024-11-21T00:26:43,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40134 deadline: 1732148863748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:43,754 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:43,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3a19beee01fe4c238917abfb77831938, entries=150, sequenceid=384, filesize=12.0 K 2024-11-21T00:26:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:43,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/d3016dc2d2544e50b5be03deb5bd1903 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d3016dc2d2544e50b5be03deb5bd1903 2024-11-21T00:26:43,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d3016dc2d2544e50b5be03deb5bd1903, entries=150, sequenceid=384, filesize=12.0 K 2024-11-21T00:26:43,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b03d35ead3c53e10e140da78ec4bd637 in 635ms, sequenceid=384, compaction requested=true 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:26:43,773 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:43,773 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b03d35ead3c53e10e140da78ec4bd637:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:26:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:43,775 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:26:43,775 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49950 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:26:43,775 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/A is initiating minor compaction (all files) 2024-11-21T00:26:43,775 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/B is initiating minor compaction (all files) 2024-11-21T00:26:43,775 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/B in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:43,775 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e4b3ab85b0ed445fa25900445cd6e29c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/29da274bf7004455b7ab7dd1abcffa7e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ded20427d6fe4536b594db0e3c471f76, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3a19beee01fe4c238917abfb77831938] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=46.4 K 2024-11-21T00:26:43,775 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/A in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:43,776 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/fea5a71d6a684d02ac09a5d754f0d5fa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/22d05eb4a6774f9eb411fcd4bf8a198b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e0d9f87782e54a80ab324fbaae606239, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/dd2e153bb9d84f7cbcbd74a73a9f18db] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=48.8 K 2024-11-21T00:26:43,776 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fea5a71d6a684d02ac09a5d754f0d5fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148798629 2024-11-21T00:26:43,776 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b3ab85b0ed445fa25900445cd6e29c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148798629 2024-11-21T00:26:43,777 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d05eb4a6774f9eb411fcd4bf8a198b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732148798788 2024-11-21T00:26:43,777 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 29da274bf7004455b7ab7dd1abcffa7e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732148798788 2024-11-21T00:26:43,777 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d9f87782e54a80ab324fbaae606239, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732148800163 2024-11-21T00:26:43,777 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd2e153bb9d84f7cbcbd74a73a9f18db, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148801934 2024-11-21T00:26:43,778 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ded20427d6fe4536b594db0e3c471f76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732148800163 2024-11-21T00:26:43,778 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a19beee01fe4c238917abfb77831938, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148803116 2024-11-21T00:26:43,806 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:43,808 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/e7e345b0caea4e4386a2538d9a09ff46 is 50, key is test_row_0/B:col10/1732148803135/Put/seqid=0 2024-11-21T00:26:43,809 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#A#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:43,810 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e851c39e4dfa42d984b3a4202ae02d75 is 50, key is test_row_0/A:col10/1732148803135/Put/seqid=0 2024-11-21T00:26:43,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741915_1091 (size=13187) 2024-11-21T00:26:43,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741914_1090 (size=13187) 2024-11-21T00:26:43,846 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/e851c39e4dfa42d984b3a4202ae02d75 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e851c39e4dfa42d984b3a4202ae02d75 2024-11-21T00:26:43,856 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/A of b03d35ead3c53e10e140da78ec4bd637 into e851c39e4dfa42d984b3a4202ae02d75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:43,856 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:43,856 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/A, priority=12, startTime=1732148803773; duration=0sec 2024-11-21T00:26:43,856 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:26:43,856 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:A 2024-11-21T00:26:43,856 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:26:43,858 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59743 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:26:43,858 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): b03d35ead3c53e10e140da78ec4bd637/C is initiating minor compaction (all files) 2024-11-21T00:26:43,858 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b03d35ead3c53e10e140da78ec4bd637/C in TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:43,858 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8c91c19d54114e7eaea89f0c8ecde945, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/897e0a56ac024eb88686b4558ac334de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/b6dba80880584786b55bab6e463f7ef3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c147f38cfa314119982a8fb13fdad87e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d3016dc2d2544e50b5be03deb5bd1903] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp, totalSize=58.3 K 2024-11-21T00:26:43,860 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c91c19d54114e7eaea89f0c8ecde945, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732148797781 2024-11-21T00:26:43,861 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 897e0a56ac024eb88686b4558ac334de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148798629 2024-11-21T00:26:43,861 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6dba80880584786b55bab6e463f7ef3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732148798788 2024-11-21T00:26:43,861 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c147f38cfa314119982a8fb13fdad87e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732148800163 2024-11-21T00:26:43,862 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3016dc2d2544e50b5be03deb5bd1903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148803116 2024-11-21T00:26:43,903 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d35ead3c53e10e140da78ec4bd637#C#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:26:43,903 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c3654c2cfae8451d99efc09ac2b450ec is 50, key is test_row_0/C:col10/1732148803135/Put/seqid=0 2024-11-21T00:26:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T00:26:43,913 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:43,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741916_1092 (size=13153) 2024-11-21T00:26:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T00:26:43,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:43,920 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:43,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:43,927 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/c3654c2cfae8451d99efc09ac2b450ec as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3654c2cfae8451d99efc09ac2b450ec 2024-11-21T00:26:43,948 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/C of b03d35ead3c53e10e140da78ec4bd637 into c3654c2cfae8451d99efc09ac2b450ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:43,948 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:43,948 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/C, priority=11, startTime=1732148803773; duration=0sec 2024-11-21T00:26:43,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:43,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:C 2024-11-21T00:26:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:44,022 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:44,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:26:44,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/7ba0ec87dc784815973dbed92ce7f38e is 50, key is test_row_0/A:col10/1732148803354/Put/seqid=0 2024-11-21T00:26:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148864068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741917_1093 (size=14741) 2024-11-21T00:26:44,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148864192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:44,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,263 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/e7e345b0caea4e4386a2538d9a09ff46 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e7e345b0caea4e4386a2538d9a09ff46 2024-11-21T00:26:44,275 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b03d35ead3c53e10e140da78ec4bd637/B of b03d35ead3c53e10e140da78ec4bd637 into e7e345b0caea4e4386a2538d9a09ff46(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:26:44,275 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:44,275 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., storeName=b03d35ead3c53e10e140da78ec4bd637/B, priority=12, startTime=1732148803773; duration=0sec 2024-11-21T00:26:44,275 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:26:44,275 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d35ead3c53e10e140da78ec4bd637:B 2024-11-21T00:26:44,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:44,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148864402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40200 deadline: 1732148864422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,428 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40174 deadline: 1732148864432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40138 deadline: 1732148864432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,440 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,441 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in b03d35ead3c53e10e140da78ec4bd637 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:26:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/7ba0ec87dc784815973dbed92ce7f38e 2024-11-21T00:26:44,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/6d636e9d00fb4474865f0317362f5fd6 is 50, key is test_row_0/B:col10/1732148803354/Put/seqid=0 2024-11-21T00:26:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741918_1094 (size=12301) 2024-11-21T00:26:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:44,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:44,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148864712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:44,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:44,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:44,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/6d636e9d00fb4474865f0317362f5fd6 2024-11-21T00:26:44,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3788cd8971134f789ae45a8753eb7c1a is 50, key is test_row_0/C:col10/1732148803354/Put/seqid=0 2024-11-21T00:26:44,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741919_1095 (size=12301) 2024-11-21T00:26:45,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:45,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:45,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40154 deadline: 1732148865232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:26:45,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:45,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3788cd8971134f789ae45a8753eb7c1a 2024-11-21T00:26:45,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/7ba0ec87dc784815973dbed92ce7f38e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/7ba0ec87dc784815973dbed92ce7f38e 2024-11-21T00:26:45,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/7ba0ec87dc784815973dbed92ce7f38e, entries=200, sequenceid=412, filesize=14.4 K 2024-11-21T00:26:45,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/6d636e9d00fb4474865f0317362f5fd6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/6d636e9d00fb4474865f0317362f5fd6 2024-11-21T00:26:45,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/6d636e9d00fb4474865f0317362f5fd6, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T00:26:45,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3788cd8971134f789ae45a8753eb7c1a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3788cd8971134f789ae45a8753eb7c1a 2024-11-21T00:26:45,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3788cd8971134f789ae45a8753eb7c1a, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T00:26:45,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b03d35ead3c53e10e140da78ec4bd637 in 1408ms, sequenceid=412, compaction requested=false 2024-11-21T00:26:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:45,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T00:26:45,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:45,486 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T00:26:45,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/d4812749751e4f4b900dc05d4675c41a is 50, key is test_row_0/A:col10/1732148804057/Put/seqid=0 2024-11-21T00:26:45,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741920_1096 (size=12301) 2024-11-21T00:26:45,765 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T00:26:45,765 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:45,766 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T00:26:45,766 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:45,773 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,773 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,774 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,776 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T00:26:45,776 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:45,776 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,776 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T00:26:45,776 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,776 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:45,776 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,776 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,777 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:45,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:45,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:45,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:45,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:45,944 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/d4812749751e4f4b900dc05d4675c41a 2024-11-21T00:26:45,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/970238ad1da6483b97dcc49586a007f4 is 50, key is test_row_0/B:col10/1732148804057/Put/seqid=0 2024-11-21T00:26:45,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741921_1097 (size=12301) 2024-11-21T00:26:45,987 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/970238ad1da6483b97dcc49586a007f4 2024-11-21T00:26:46,006 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:26:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8ebb992a04cf412cac32f773954a8d1a is 50, key is test_row_0/C:col10/1732148804057/Put/seqid=0 2024-11-21T00:26:46,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741922_1098 (size=12301) 2024-11-21T00:26:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. as already flushing 2024-11-21T00:26:46,248 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T00:26:46,248 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:46,248 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:46,249 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:46,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:46,444 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8ebb992a04cf412cac32f773954a8d1a 2024-11-21T00:26:46,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/d4812749751e4f4b900dc05d4675c41a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/d4812749751e4f4b900dc05d4675c41a 2024-11-21T00:26:46,465 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/d4812749751e4f4b900dc05d4675c41a, entries=150, sequenceid=423, filesize=12.0 K 2024-11-21T00:26:46,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/970238ad1da6483b97dcc49586a007f4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/970238ad1da6483b97dcc49586a007f4 2024-11-21T00:26:46,474 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/970238ad1da6483b97dcc49586a007f4, entries=150, sequenceid=423, filesize=12.0 K 2024-11-21T00:26:46,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/8ebb992a04cf412cac32f773954a8d1a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ebb992a04cf412cac32f773954a8d1a 2024-11-21T00:26:46,487 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ebb992a04cf412cac32f773954a8d1a, entries=150, sequenceid=423, filesize=12.0 K 2024-11-21T00:26:46,488 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=6.71 KB/6870 for b03d35ead3c53e10e140da78ec4bd637 in 1002ms, sequenceid=423, compaction requested=true 2024-11-21T00:26:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b03d35ead3c53e10e140da78ec4bd637: 2024-11-21T00:26:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:46,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T00:26:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T00:26:46,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T00:26:46,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5740 sec 2024-11-21T00:26:46,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.5830 sec 2024-11-21T00:26:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T00:26:48,093 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:48,446 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T00:26:48,446 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:48,446 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:48,446 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:48,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:48,454 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T00:26:48,455 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:48,455 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:48,455 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:48,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:48,455 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T00:26:48,455 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:48,455 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:48,456 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:48,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:53,774 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T00:26:53,774 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:26:53,774 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:53,774 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:53,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1865 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1887 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 941 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2823 rows 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 934 2024-11-21T00:26:53,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2797 rows 2024-11-21T00:26:53,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:26:53,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:26:53,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:53,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:53,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:26:53,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:26:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:26:53,789 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148813789"}]},"ts":"1732148813789"} 2024-11-21T00:26:53,791 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:26:53,791 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:26:53,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:26:53,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, UNASSIGN}] 2024-11-21T00:26:53,799 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, UNASSIGN 2024-11-21T00:26:53,800 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b03d35ead3c53e10e140da78ec4bd637, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:53,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, UNASSIGN because future has completed 2024-11-21T00:26:53,803 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:26:53,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:26:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:26:53,960 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:53,960 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:26:53,961 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing b03d35ead3c53e10e140da78ec4bd637, disabling compactions & flushes 2024-11-21T00:26:53,961 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:53,961 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:53,961 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. after waiting 0 ms 2024-11-21T00:26:53,961 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:53,961 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing b03d35ead3c53e10e140da78ec4bd637 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T00:26:53,966 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/796a083367024d929e2e2529fa7abb80 is 50, key is test_row_0/A:col10/1732148808444/Put/seqid=0 2024-11-21T00:26:53,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741923_1099 (size=12301) 2024-11-21T00:26:53,971 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/796a083367024d929e2e2529fa7abb80 2024-11-21T00:26:53,979 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/1d06a14e80ee4f93926cb41a505d7f5c is 50, key is test_row_0/B:col10/1732148808444/Put/seqid=0 2024-11-21T00:26:53,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741924_1100 (size=12301) 2024-11-21T00:26:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:26:54,385 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/1d06a14e80ee4f93926cb41a505d7f5c 2024-11-21T00:26:54,393 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3e17e65109d1494bbef6b0d90d90633e is 50, key is test_row_0/C:col10/1732148808444/Put/seqid=0 2024-11-21T00:26:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741925_1101 (size=12301) 2024-11-21T00:26:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:26:54,798 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3e17e65109d1494bbef6b0d90d90633e 2024-11-21T00:26:54,805 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/A/796a083367024d929e2e2529fa7abb80 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/796a083367024d929e2e2529fa7abb80 2024-11-21T00:26:54,810 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/796a083367024d929e2e2529fa7abb80, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:26:54,811 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/B/1d06a14e80ee4f93926cb41a505d7f5c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/1d06a14e80ee4f93926cb41a505d7f5c 2024-11-21T00:26:54,817 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/1d06a14e80ee4f93926cb41a505d7f5c, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:26:54,818 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/.tmp/C/3e17e65109d1494bbef6b0d90d90633e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3e17e65109d1494bbef6b0d90d90633e 2024-11-21T00:26:54,824 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3e17e65109d1494bbef6b0d90d90633e, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:26:54,825 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b03d35ead3c53e10e140da78ec4bd637 in 864ms, sequenceid=431, compaction requested=true 2024-11-21T00:26:54,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/bc651fc895ff408dad69028107d19181, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/27d3ad216e5242c6b7a409582675e683, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/1350a93c5d8b4469ba5d1206d87b1239, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/964c40e5f8ee40d0afa021ac00977ec3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f82e3e0acd744e4bbc2d17568bf8e863, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/0d13046e15e74b34a88685ff59a5e046, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/60a39cae9f1d462495699ec327f1fee3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/49723f20877f4210a82ab7447ab62c64, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/b339b91e9b1e48ffae1b34a01f35870a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/9573d22b2d544561969d3c7df6de5546, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e5e70c6295644ac888c4b75253fbb0d0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/51968439db5943caa379b5ae61549a42, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f20a04728abb4baa994d4329038fb142, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/8a092521c40e43ae9fe2fc1961a73250, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/84049daad1084eaeafb46c838c8863b9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/ab4dda363a6b41d0a7f51bf434ccfd0c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/5533a9db3cc84d698b549c8711428aa6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e44ec91b9ac844d6af88494516e5bf52, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/716c93bb7a024f26b7c314e9f1401fb6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/45e0594d220441cb8757f7fee4f21754, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/770627a2dd38493a8daa84515a382003, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/32b6d46e3e654eeaa1580c6252a6abff, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/fea5a71d6a684d02ac09a5d754f0d5fa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/22d05eb4a6774f9eb411fcd4bf8a198b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e0d9f87782e54a80ab324fbaae606239, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/dd2e153bb9d84f7cbcbd74a73a9f18db] to archive 2024-11-21T00:26:54,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:26:54,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/bc651fc895ff408dad69028107d19181 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/bc651fc895ff408dad69028107d19181 2024-11-21T00:26:54,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/27d3ad216e5242c6b7a409582675e683 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/27d3ad216e5242c6b7a409582675e683 2024-11-21T00:26:54,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/1350a93c5d8b4469ba5d1206d87b1239 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/1350a93c5d8b4469ba5d1206d87b1239 2024-11-21T00:26:54,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/964c40e5f8ee40d0afa021ac00977ec3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/964c40e5f8ee40d0afa021ac00977ec3 2024-11-21T00:26:54,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f82e3e0acd744e4bbc2d17568bf8e863 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f82e3e0acd744e4bbc2d17568bf8e863 2024-11-21T00:26:54,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/0d13046e15e74b34a88685ff59a5e046 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/0d13046e15e74b34a88685ff59a5e046 2024-11-21T00:26:54,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/60a39cae9f1d462495699ec327f1fee3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/60a39cae9f1d462495699ec327f1fee3 2024-11-21T00:26:54,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/49723f20877f4210a82ab7447ab62c64 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/49723f20877f4210a82ab7447ab62c64 2024-11-21T00:26:54,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/b339b91e9b1e48ffae1b34a01f35870a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/b339b91e9b1e48ffae1b34a01f35870a 2024-11-21T00:26:54,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/9573d22b2d544561969d3c7df6de5546 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/9573d22b2d544561969d3c7df6de5546 2024-11-21T00:26:54,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e5e70c6295644ac888c4b75253fbb0d0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e5e70c6295644ac888c4b75253fbb0d0 2024-11-21T00:26:54,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/51968439db5943caa379b5ae61549a42 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/51968439db5943caa379b5ae61549a42 2024-11-21T00:26:54,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f20a04728abb4baa994d4329038fb142 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/f20a04728abb4baa994d4329038fb142 2024-11-21T00:26:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:26:55,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/8a092521c40e43ae9fe2fc1961a73250 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/8a092521c40e43ae9fe2fc1961a73250 2024-11-21T00:26:55,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/84049daad1084eaeafb46c838c8863b9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/84049daad1084eaeafb46c838c8863b9 2024-11-21T00:26:55,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/ab4dda363a6b41d0a7f51bf434ccfd0c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/ab4dda363a6b41d0a7f51bf434ccfd0c 2024-11-21T00:26:55,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/5533a9db3cc84d698b549c8711428aa6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/5533a9db3cc84d698b549c8711428aa6 2024-11-21T00:26:55,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e44ec91b9ac844d6af88494516e5bf52 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e44ec91b9ac844d6af88494516e5bf52 2024-11-21T00:26:55,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/716c93bb7a024f26b7c314e9f1401fb6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/716c93bb7a024f26b7c314e9f1401fb6 2024-11-21T00:26:55,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/45e0594d220441cb8757f7fee4f21754 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/45e0594d220441cb8757f7fee4f21754 2024-11-21T00:26:55,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/770627a2dd38493a8daa84515a382003 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/770627a2dd38493a8daa84515a382003 2024-11-21T00:26:55,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/32b6d46e3e654eeaa1580c6252a6abff to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/32b6d46e3e654eeaa1580c6252a6abff 2024-11-21T00:26:55,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/fea5a71d6a684d02ac09a5d754f0d5fa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/fea5a71d6a684d02ac09a5d754f0d5fa 2024-11-21T00:26:55,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/22d05eb4a6774f9eb411fcd4bf8a198b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/22d05eb4a6774f9eb411fcd4bf8a198b 2024-11-21T00:26:55,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e0d9f87782e54a80ab324fbaae606239 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e0d9f87782e54a80ab324fbaae606239 2024-11-21T00:26:55,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/dd2e153bb9d84f7cbcbd74a73a9f18db to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/dd2e153bb9d84f7cbcbd74a73a9f18db 2024-11-21T00:26:55,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8038f3c60a4c47a99e962a21e2a611c5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/d1fa11e5d5d2410d92ccd795512682ea, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/5f536cb5ee4e4d26b9b097fb63b48574, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ea582647b9aa4d479b600faeb3fb7b53, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/066a5683a8664e2887aa63b15655dfb0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/371f946d733e440988253c660dba4a14, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/4065282fffb647c8aaddfb5d6a0c92b9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fdfd580bfad5464abc4ef19806f0553b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7e5dffed60d94abbac4d98e331da8e0d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8371351a2cb44bc89ca732c4a34573e1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3c52e30664ec48c7877a2ca39a5dffa0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3ab1db70f3eb4747a89e91fce1882ebc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/b35d20680ce643278c00c3e705947ddc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/46a7d1baa5ad4db9a5f5ffbe67d23b87, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/04209edc23944e2a89d0c84ae9629f13, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/db1b46e295ff48d898ea37d854b0812c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/beb9f220e4134b05bf7e14d41b9ee854, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3623651ed14d4541bef0f851aadfeec9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fd3e2ea8941c4f5fb097739bd75c0869, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/bc8a6b65521240dc97ae9517e0fb3297, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/0e55bf62ccb6439fb57150914b172c0f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e4b3ab85b0ed445fa25900445cd6e29c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7663a66bfa1944caa4fe86fcff6e07d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/29da274bf7004455b7ab7dd1abcffa7e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ded20427d6fe4536b594db0e3c471f76, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3a19beee01fe4c238917abfb77831938] to archive 2024-11-21T00:26:55,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:26:55,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8038f3c60a4c47a99e962a21e2a611c5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8038f3c60a4c47a99e962a21e2a611c5 2024-11-21T00:26:55,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/d1fa11e5d5d2410d92ccd795512682ea to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/d1fa11e5d5d2410d92ccd795512682ea 2024-11-21T00:26:55,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/5f536cb5ee4e4d26b9b097fb63b48574 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/5f536cb5ee4e4d26b9b097fb63b48574 2024-11-21T00:26:55,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ea582647b9aa4d479b600faeb3fb7b53 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ea582647b9aa4d479b600faeb3fb7b53 2024-11-21T00:26:55,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/066a5683a8664e2887aa63b15655dfb0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/066a5683a8664e2887aa63b15655dfb0 2024-11-21T00:26:55,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/371f946d733e440988253c660dba4a14 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/371f946d733e440988253c660dba4a14 2024-11-21T00:26:55,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/4065282fffb647c8aaddfb5d6a0c92b9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/4065282fffb647c8aaddfb5d6a0c92b9 2024-11-21T00:26:55,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fdfd580bfad5464abc4ef19806f0553b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fdfd580bfad5464abc4ef19806f0553b 2024-11-21T00:26:55,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7e5dffed60d94abbac4d98e331da8e0d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7e5dffed60d94abbac4d98e331da8e0d 2024-11-21T00:26:55,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8371351a2cb44bc89ca732c4a34573e1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/8371351a2cb44bc89ca732c4a34573e1 2024-11-21T00:26:55,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3c52e30664ec48c7877a2ca39a5dffa0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3c52e30664ec48c7877a2ca39a5dffa0 2024-11-21T00:26:55,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3ab1db70f3eb4747a89e91fce1882ebc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3ab1db70f3eb4747a89e91fce1882ebc 2024-11-21T00:26:55,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/b35d20680ce643278c00c3e705947ddc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/b35d20680ce643278c00c3e705947ddc 2024-11-21T00:26:55,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/46a7d1baa5ad4db9a5f5ffbe67d23b87 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/46a7d1baa5ad4db9a5f5ffbe67d23b87 2024-11-21T00:26:55,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/04209edc23944e2a89d0c84ae9629f13 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/04209edc23944e2a89d0c84ae9629f13 2024-11-21T00:26:55,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/db1b46e295ff48d898ea37d854b0812c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/db1b46e295ff48d898ea37d854b0812c 2024-11-21T00:26:55,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/beb9f220e4134b05bf7e14d41b9ee854 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/beb9f220e4134b05bf7e14d41b9ee854 2024-11-21T00:26:55,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3623651ed14d4541bef0f851aadfeec9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3623651ed14d4541bef0f851aadfeec9 2024-11-21T00:26:55,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fd3e2ea8941c4f5fb097739bd75c0869 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/fd3e2ea8941c4f5fb097739bd75c0869 2024-11-21T00:26:55,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/bc8a6b65521240dc97ae9517e0fb3297 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/bc8a6b65521240dc97ae9517e0fb3297 2024-11-21T00:26:55,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/0e55bf62ccb6439fb57150914b172c0f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/0e55bf62ccb6439fb57150914b172c0f 2024-11-21T00:26:55,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e4b3ab85b0ed445fa25900445cd6e29c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e4b3ab85b0ed445fa25900445cd6e29c 2024-11-21T00:26:55,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7663a66bfa1944caa4fe86fcff6e07d1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/7663a66bfa1944caa4fe86fcff6e07d1 2024-11-21T00:26:55,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/29da274bf7004455b7ab7dd1abcffa7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/29da274bf7004455b7ab7dd1abcffa7e 2024-11-21T00:26:55,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ded20427d6fe4536b594db0e3c471f76 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/ded20427d6fe4536b594db0e3c471f76 2024-11-21T00:26:55,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3a19beee01fe4c238917abfb77831938 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/3a19beee01fe4c238917abfb77831938 2024-11-21T00:26:55,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/25842dcd72684aba84547dadbeed323b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d8e8237faed64c71bc28a099871f8c4c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/6bd57be4522f4c08bda918cda990c9a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d21a3ce691d842e2b229815c2aaae993, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/e46f28e57000460d8ab8dfd2c6d4052a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/47ad3b2ba0a94be3bb9c3805f31d1610, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/2d4483b301b045de9df3eee3d4930f75, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4cdbe562b5784b569e68392899bf3363, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ce6fa33a1df4b97900a2c232d2f5621, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4aaae6f9748a41c0a1369b19ed196e86, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/530e3608d00a4e698d490eb6fa263209, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/bb669373b41d48379c0a2b00a8d97fde, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3b94b11f3e1b428ba768d9adaa91e991, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5ba8e2ee04dd4c8c8660bfe6293f312d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/519366c35aa442a4bd545616d03559ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d6027366748641d596353526e8b8980e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/892337f436e747ed96adf9bb77fb96e4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3b85ed10d6749ddbea1f08a360090a0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/1df42d42a25a49a4936e3f87321e5a42, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8c91c19d54114e7eaea89f0c8ecde945, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5be8012f913840f897eb9424b8268e37, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/897e0a56ac024eb88686b4558ac334de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/b6dba80880584786b55bab6e463f7ef3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c147f38cfa314119982a8fb13fdad87e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d3016dc2d2544e50b5be03deb5bd1903] to archive 2024-11-21T00:26:55,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:26:55,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/25842dcd72684aba84547dadbeed323b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/25842dcd72684aba84547dadbeed323b 2024-11-21T00:26:55,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d8e8237faed64c71bc28a099871f8c4c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d8e8237faed64c71bc28a099871f8c4c 2024-11-21T00:26:55,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/6bd57be4522f4c08bda918cda990c9a6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/6bd57be4522f4c08bda918cda990c9a6 2024-11-21T00:26:55,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d21a3ce691d842e2b229815c2aaae993 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d21a3ce691d842e2b229815c2aaae993 2024-11-21T00:26:55,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/e46f28e57000460d8ab8dfd2c6d4052a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/e46f28e57000460d8ab8dfd2c6d4052a 2024-11-21T00:26:55,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/47ad3b2ba0a94be3bb9c3805f31d1610 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/47ad3b2ba0a94be3bb9c3805f31d1610 2024-11-21T00:26:55,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/2d4483b301b045de9df3eee3d4930f75 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/2d4483b301b045de9df3eee3d4930f75 2024-11-21T00:26:55,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4cdbe562b5784b569e68392899bf3363 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4cdbe562b5784b569e68392899bf3363 2024-11-21T00:26:55,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ce6fa33a1df4b97900a2c232d2f5621 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ce6fa33a1df4b97900a2c232d2f5621 2024-11-21T00:26:55,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4aaae6f9748a41c0a1369b19ed196e86 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/4aaae6f9748a41c0a1369b19ed196e86 2024-11-21T00:26:55,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/530e3608d00a4e698d490eb6fa263209 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/530e3608d00a4e698d490eb6fa263209 2024-11-21T00:26:55,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/bb669373b41d48379c0a2b00a8d97fde to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/bb669373b41d48379c0a2b00a8d97fde 2024-11-21T00:26:55,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3b94b11f3e1b428ba768d9adaa91e991 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3b94b11f3e1b428ba768d9adaa91e991 2024-11-21T00:26:55,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5ba8e2ee04dd4c8c8660bfe6293f312d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5ba8e2ee04dd4c8c8660bfe6293f312d 2024-11-21T00:26:55,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/519366c35aa442a4bd545616d03559ee to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/519366c35aa442a4bd545616d03559ee 2024-11-21T00:26:55,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d6027366748641d596353526e8b8980e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d6027366748641d596353526e8b8980e 2024-11-21T00:26:55,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/892337f436e747ed96adf9bb77fb96e4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/892337f436e747ed96adf9bb77fb96e4 2024-11-21T00:26:55,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3b85ed10d6749ddbea1f08a360090a0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3b85ed10d6749ddbea1f08a360090a0 2024-11-21T00:26:55,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/1df42d42a25a49a4936e3f87321e5a42 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/1df42d42a25a49a4936e3f87321e5a42 2024-11-21T00:26:55,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8c91c19d54114e7eaea89f0c8ecde945 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8c91c19d54114e7eaea89f0c8ecde945 2024-11-21T00:26:55,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5be8012f913840f897eb9424b8268e37 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/5be8012f913840f897eb9424b8268e37 2024-11-21T00:26:55,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/897e0a56ac024eb88686b4558ac334de to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/897e0a56ac024eb88686b4558ac334de 2024-11-21T00:26:55,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/b6dba80880584786b55bab6e463f7ef3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/b6dba80880584786b55bab6e463f7ef3 2024-11-21T00:26:55,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c147f38cfa314119982a8fb13fdad87e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c147f38cfa314119982a8fb13fdad87e 2024-11-21T00:26:55,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d3016dc2d2544e50b5be03deb5bd1903 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/d3016dc2d2544e50b5be03deb5bd1903 2024-11-21T00:26:55,212 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=1 2024-11-21T00:26:55,213 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. 2024-11-21T00:26:55,213 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for b03d35ead3c53e10e140da78ec4bd637: Waiting for close lock at 1732148813961Running coprocessor pre-close hooks at 1732148813961Disabling compacts and flushes for region at 1732148813961Disabling writes for close at 1732148813961Obtaining lock to block concurrent updates at 1732148813961Preparing flush snapshotting stores in b03d35ead3c53e10e140da78ec4bd637 at 1732148813961Finished memstore snapshotting TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732148813961Flushing stores of TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637. at 1732148813962 (+1 ms)Flushing b03d35ead3c53e10e140da78ec4bd637/A: creating writer at 1732148813962Flushing b03d35ead3c53e10e140da78ec4bd637/A: appending metadata at 1732148813966 (+4 ms)Flushing b03d35ead3c53e10e140da78ec4bd637/A: closing flushed file at 1732148813966Flushing b03d35ead3c53e10e140da78ec4bd637/B: creating writer at 1732148813976 (+10 ms)Flushing b03d35ead3c53e10e140da78ec4bd637/B: appending metadata at 1732148813979 (+3 ms)Flushing b03d35ead3c53e10e140da78ec4bd637/B: closing flushed file at 1732148813979Flushing b03d35ead3c53e10e140da78ec4bd637/C: creating writer at 1732148814390 (+411 ms)Flushing b03d35ead3c53e10e140da78ec4bd637/C: appending metadata at 1732148814393 (+3 ms)Flushing b03d35ead3c53e10e140da78ec4bd637/C: closing flushed file at 1732148814393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d360741: reopening flushed file at 1732148814803 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bac5fc2: reopening flushed file at 1732148814810 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f91115a: reopening flushed file at 1732148814817 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b03d35ead3c53e10e140da78ec4bd637 in 864ms, sequenceid=431, compaction requested=true at 1732148814825 (+8 ms)Writing region close event to WAL at 1732148815208 (+383 ms)Running coprocessor post-close hooks at 1732148815213 (+5 ms)Closed at 1732148815213 2024-11-21T00:26:55,216 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:55,216 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b03d35ead3c53e10e140da78ec4bd637, regionState=CLOSED 2024-11-21T00:26:55,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:26:55,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T00:26:55,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure b03d35ead3c53e10e140da78ec4bd637, server=5ed4808ef0e6,38589,1732148778087 in 1.4170 sec 2024-11-21T00:26:55,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-21T00:26:55,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b03d35ead3c53e10e140da78ec4bd637, UNASSIGN in 1.4250 sec 2024-11-21T00:26:55,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T00:26:55,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4330 sec 2024-11-21T00:26:55,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148815229"}]},"ts":"1732148815229"} 2024-11-21T00:26:55,231 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:26:55,231 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:26:55,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4500 sec 2024-11-21T00:26:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T00:26:55,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:55,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:55,942 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:26:55,944 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:55,948 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:55,952 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/recovered.edits] 2024-11-21T00:26:55,956 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/796a083367024d929e2e2529fa7abb80 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/796a083367024d929e2e2529fa7abb80 2024-11-21T00:26:55,958 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/7ba0ec87dc784815973dbed92ce7f38e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/7ba0ec87dc784815973dbed92ce7f38e 2024-11-21T00:26:55,959 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/d4812749751e4f4b900dc05d4675c41a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/d4812749751e4f4b900dc05d4675c41a 2024-11-21T00:26:55,960 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e851c39e4dfa42d984b3a4202ae02d75 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/A/e851c39e4dfa42d984b3a4202ae02d75 2024-11-21T00:26:55,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/1d06a14e80ee4f93926cb41a505d7f5c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/1d06a14e80ee4f93926cb41a505d7f5c 2024-11-21T00:26:55,967 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/6d636e9d00fb4474865f0317362f5fd6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/6d636e9d00fb4474865f0317362f5fd6 2024-11-21T00:26:55,969 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/970238ad1da6483b97dcc49586a007f4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/970238ad1da6483b97dcc49586a007f4 2024-11-21T00:26:55,970 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e7e345b0caea4e4386a2538d9a09ff46 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/B/e7e345b0caea4e4386a2538d9a09ff46 2024-11-21T00:26:55,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3788cd8971134f789ae45a8753eb7c1a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3788cd8971134f789ae45a8753eb7c1a 2024-11-21T00:26:55,975 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3e17e65109d1494bbef6b0d90d90633e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/3e17e65109d1494bbef6b0d90d90633e 2024-11-21T00:26:55,977 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ebb992a04cf412cac32f773954a8d1a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/8ebb992a04cf412cac32f773954a8d1a 2024-11-21T00:26:55,979 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3654c2cfae8451d99efc09ac2b450ec to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/C/c3654c2cfae8451d99efc09ac2b450ec 2024-11-21T00:26:55,982 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/recovered.edits/434.seqid to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637/recovered.edits/434.seqid 2024-11-21T00:26:55,983 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/b03d35ead3c53e10e140da78ec4bd637 2024-11-21T00:26:55,983 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:26:55,989 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:55,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T00:26:55,998 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:26:56,033 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:26:56,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:56,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:26:56,037 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148816036"}]},"ts":"9223372036854775807"} 2024-11-21T00:26:56,048 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:26:56,048 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b03d35ead3c53e10e140da78ec4bd637, NAME => 'TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:26:56,049 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:26:56,049 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148816049"}]},"ts":"9223372036854775807"} 2024-11-21T00:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:26:56,054 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:26:56,055 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:56,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 118 msec 2024-11-21T00:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T00:26:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:26:56,263 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:56,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;5ed4808ef0e6:38589-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-105637966_22 at /127.0.0.1:59174 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=825 (was 645) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2040 (was 1963) - AvailableMemoryMB LEAK? - 2024-11-21T00:26:56,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=825, ProcessCount=11, AvailableMemoryMB=2038 2024-11-21T00:26:56,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:26:56,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:26:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:56,291 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:26:56,291 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:56,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-21T00:26:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T00:26:56,293 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:26:56,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741926_1102 (size=959) 2024-11-21T00:26:56,303 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:26:56,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741927_1103 (size=53) 2024-11-21T00:26:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T00:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T00:26:56,717 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:56,717 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d3834c10900814f7a41633aca86edf7e, disabling compactions & flushes 2024-11-21T00:26:56,717 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:56,721 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:56,721 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. after waiting 0 ms 2024-11-21T00:26:56,721 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:56,721 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:56,721 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3834c10900814f7a41633aca86edf7e: Waiting for close lock at 1732148816717Disabling compacts and flushes for region at 1732148816717Disabling writes for close at 1732148816721 (+4 ms)Writing region close event to WAL at 1732148816721Closed at 1732148816721 2024-11-21T00:26:56,723 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:26:56,723 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148816723"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148816723"}]},"ts":"1732148816723"} 2024-11-21T00:26:56,728 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:26:56,729 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:26:56,730 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148816730"}]},"ts":"1732148816730"} 2024-11-21T00:26:56,732 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:26:56,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, ASSIGN}] 2024-11-21T00:26:56,740 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, ASSIGN 2024-11-21T00:26:56,742 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=false 2024-11-21T00:26:56,893 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:56,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, ASSIGN because future has completed 2024-11-21T00:26:56,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:26:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T00:26:57,066 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,066 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:26:57,067 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,067 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:57,067 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,067 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,076 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,086 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3834c10900814f7a41633aca86edf7e columnFamilyName A 2024-11-21T00:26:57,087 DEBUG [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:57,087 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(327): Store=d3834c10900814f7a41633aca86edf7e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:57,088 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,089 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3834c10900814f7a41633aca86edf7e columnFamilyName B 2024-11-21T00:26:57,089 DEBUG [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:57,090 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(327): Store=d3834c10900814f7a41633aca86edf7e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:57,090 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,092 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3834c10900814f7a41633aca86edf7e columnFamilyName C 2024-11-21T00:26:57,092 DEBUG [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:57,093 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(327): Store=d3834c10900814f7a41633aca86edf7e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:57,093 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,094 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,095 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,096 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,096 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,097 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:26:57,098 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,111 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:26:57,111 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened d3834c10900814f7a41633aca86edf7e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67777790, jitterRate=0.009967774152755737}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:26:57,112 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,113 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for d3834c10900814f7a41633aca86edf7e: Running coprocessor pre-open hook at 1732148817067Writing region info on filesystem at 1732148817067Initializing all the Stores at 1732148817069 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148817069Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148817072 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148817072Cleaning up temporary data from old regions at 1732148817096 (+24 ms)Running coprocessor post-open hooks at 1732148817112 (+16 ms)Region opened successfully at 1732148817113 (+1 ms) 2024-11-21T00:26:57,115 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., pid=30, masterSystemTime=1732148817060 2024-11-21T00:26:57,130 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,130 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,137 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:57,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:26:57,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-21T00:26:57,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, ASSIGN in 428 msec 2024-11-21T00:26:57,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T00:26:57,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 in 254 msec 2024-11-21T00:26:57,176 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:26:57,176 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148817176"}]},"ts":"1732148817176"} 2024-11-21T00:26:57,181 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:26:57,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:26:57,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 897 msec 2024-11-21T00:26:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T00:26:57,433 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:57,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49091041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:57,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:57,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:57,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d875f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:57,460 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d4582a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:57,463 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:57,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:57,469 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:26:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:26:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56a7359c 2024-11-21T00:26:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:26:57,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:26:57,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:26:57,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:26:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T00:26:57,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741928_1104 (size=995) 2024-11-21T00:26:57,547 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T00:26:57,547 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T00:26:57,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:26:57,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, REOPEN/MOVE}] 2024-11-21T00:26:57,562 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, REOPEN/MOVE 2024-11-21T00:26:57,563 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:57,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, REOPEN/MOVE because future has completed 2024-11-21T00:26:57,592 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:26:57,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:26:57,751 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,751 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:26:57,751 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing d3834c10900814f7a41633aca86edf7e, disabling compactions & flushes 2024-11-21T00:26:57,751 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,751 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,751 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. after waiting 0 ms 2024-11-21T00:26:57,751 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,757 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T00:26:57,758 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:57,758 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for d3834c10900814f7a41633aca86edf7e: Waiting for close lock at 1732148817751Running coprocessor pre-close hooks at 1732148817751Disabling compacts and flushes for region at 1732148817751Disabling writes for close at 1732148817751Writing region close event to WAL at 1732148817752 (+1 ms)Running coprocessor post-close hooks at 1732148817758 (+6 ms)Closed at 1732148817758 2024-11-21T00:26:57,758 WARN [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: d3834c10900814f7a41633aca86edf7e to self. 2024-11-21T00:26:57,763 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:57,763 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=CLOSED 2024-11-21T00:26:57,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:26:57,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T00:26:57,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 in 176 msec 2024-11-21T00:26:57,771 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, REOPEN/MOVE; state=CLOSED, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=true 2024-11-21T00:26:57,924 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:57,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, REOPEN/MOVE because future has completed 2024-11-21T00:26:57,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:26:58,097 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,098 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:26:58,098 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,098 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:26:58,098 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,098 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,107 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,113 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3834c10900814f7a41633aca86edf7e columnFamilyName A 2024-11-21T00:26:58,115 DEBUG [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:58,116 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(327): Store=d3834c10900814f7a41633aca86edf7e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:58,116 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,120 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3834c10900814f7a41633aca86edf7e columnFamilyName B 2024-11-21T00:26:58,120 DEBUG [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:58,121 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(327): Store=d3834c10900814f7a41633aca86edf7e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:58,121 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,122 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3834c10900814f7a41633aca86edf7e columnFamilyName C 2024-11-21T00:26:58,123 DEBUG [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:58,123 INFO [StoreOpener-d3834c10900814f7a41633aca86edf7e-1 {}] regionserver.HStore(327): Store=d3834c10900814f7a41633aca86edf7e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:26:58,123 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,124 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,126 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,127 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,127 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,127 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:26:58,130 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,130 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened d3834c10900814f7a41633aca86edf7e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75101193, jitterRate=0.11909498274326324}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:26:58,131 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,133 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for d3834c10900814f7a41633aca86edf7e: Running coprocessor pre-open hook at 1732148818099Writing region info on filesystem at 1732148818099Initializing all the Stores at 1732148818100 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148818100Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148818104 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148818104Cleaning up temporary data from old regions at 1732148818127 (+23 ms)Running coprocessor post-open hooks at 1732148818131 (+4 ms)Region opened successfully at 1732148818133 (+2 ms) 2024-11-21T00:26:58,137 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., pid=35, masterSystemTime=1732148818093 2024-11-21T00:26:58,139 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,139 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,140 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=OPEN, openSeqNum=5, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:26:58,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-21T00:26:58,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 in 205 msec 2024-11-21T00:26:58,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-21T00:26:58,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, REOPEN/MOVE in 589 msec 2024-11-21T00:26:58,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 681 msec 2024-11-21T00:26:58,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-21T00:26:58,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 604 msec 2024-11-21T00:26:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T00:26:58,172 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:26:58,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cce2794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,182 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696ef75f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366f7c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,188 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b51fc47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7dd86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,201 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7c86f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,207 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0f963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,214 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd99ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,217 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f818ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,222 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e76c8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,227 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c35cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,245 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3e1796, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330e45ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b54a9e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442575c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,294 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771c18b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,296 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d44aa2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:26:58,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:26:58,301 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:26:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:26:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:26:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6152e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:26:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:26:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:26:58,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:26:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678171c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:26:58,308 DEBUG [Thread-561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,313 DEBUG [Thread-559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,338 DEBUG [Thread-563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:26:58,349 DEBUG [Thread-552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,364 DEBUG [Thread-565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,381 DEBUG [Thread-548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,385 DEBUG [Thread-556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-21T00:26:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-21T00:26:58,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:26:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:26:58,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:26:58,401 DEBUG [Thread-550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,411 DEBUG [Thread-554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:26:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:26:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:26:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:26:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:26:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:58,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:26:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-21T00:26:58,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148878521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148878521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148878523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-21T00:26:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:26:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148878531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148878531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121933ea4aaf2c9469aa30da6d96a79b625_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148818463/Put/seqid=0 2024-11-21T00:26:58,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741929_1105 (size=14594) 2024-11-21T00:26:58,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:58,605 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121933ea4aaf2c9469aa30da6d96a79b625_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121933ea4aaf2c9469aa30da6d96a79b625_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:58,608 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/71ea47706f88417f897ac03e7050dd52, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:26:58,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/71ea47706f88417f897ac03e7050dd52 is 175, key is test_row_0/A:col10/1732148818463/Put/seqid=0 2024-11-21T00:26:58,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741930_1106 (size=39549) 2024-11-21T00:26:58,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/71ea47706f88417f897ac03e7050dd52 2024-11-21T00:26:58,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148878647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148878648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148878648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148878665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148878665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-21T00:26:58,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:26:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-21T00:26:58,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/362f46cef6164b958a005dd0dc0b3d25 is 50, key is test_row_0/B:col10/1732148818463/Put/seqid=0 2024-11-21T00:26:58,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741931_1107 (size=12001) 2024-11-21T00:26:58,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/362f46cef6164b958a005dd0dc0b3d25 2024-11-21T00:26:58,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-21T00:26:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:26:58,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:58,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:26:58,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b1e448237c3b4b36aa9b09cbeb4c3aa5 is 50, key is test_row_0/C:col10/1732148818463/Put/seqid=0 2024-11-21T00:26:58,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148878872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148878873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148878882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148878883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148878883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:58,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741932_1108 (size=12001) 2024-11-21T00:26:58,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b1e448237c3b4b36aa9b09cbeb4c3aa5 2024-11-21T00:26:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/71ea47706f88417f897ac03e7050dd52 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52 2024-11-21T00:26:58,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52, entries=200, sequenceid=17, filesize=38.6 K 2024-11-21T00:26:58,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/362f46cef6164b958a005dd0dc0b3d25 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/362f46cef6164b958a005dd0dc0b3d25 2024-11-21T00:26:58,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/362f46cef6164b958a005dd0dc0b3d25, entries=150, sequenceid=17, filesize=11.7 K 2024-11-21T00:26:58,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b1e448237c3b4b36aa9b09cbeb4c3aa5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b1e448237c3b4b36aa9b09cbeb4c3aa5 2024-11-21T00:26:58,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b1e448237c3b4b36aa9b09cbeb4c3aa5, entries=150, sequenceid=17, filesize=11.7 K 2024-11-21T00:26:58,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d3834c10900814f7a41633aca86edf7e in 464ms, sequenceid=17, compaction requested=false 2024-11-21T00:26:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:26:59,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-21T00:26:59,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:59,009 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T00:26:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-21T00:26:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213aeea7e0f7a749c89a0b0ad4510f7878_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148818512/Put/seqid=0 2024-11-21T00:26:59,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741933_1109 (size=12154) 2024-11-21T00:26:59,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:26:59,144 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213aeea7e0f7a749c89a0b0ad4510f7878_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213aeea7e0f7a749c89a0b0ad4510f7878_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:59,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e6e3254485ff4aa1bbd27cc7a79da1d5, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:26:59,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e6e3254485ff4aa1bbd27cc7a79da1d5 is 175, key is test_row_0/A:col10/1732148818512/Put/seqid=0 2024-11-21T00:26:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741934_1110 (size=30955) 2024-11-21T00:26:59,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:26:59,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148879262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148879253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148879292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148879297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148879302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148879402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148879402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148879413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148879433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148879433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-21T00:26:59,621 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e6e3254485ff4aa1bbd27cc7a79da1d5 2024-11-21T00:26:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148879622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148879622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148879633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/75a91be9321f4f72b6c516af3b4bcf9d is 50, key is test_row_0/B:col10/1732148818512/Put/seqid=0 2024-11-21T00:26:59,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148879653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:26:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148879653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:26:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:26:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:26:59,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741935_1111 (size=12001) 2024-11-21T00:26:59,684 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/75a91be9321f4f72b6c516af3b4bcf9d 2024-11-21T00:26:59,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/19d2677ac9b243f0bad4ea3bac9ec4a9 is 50, key is test_row_0/C:col10/1732148818512/Put/seqid=0 2024-11-21T00:26:59,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741936_1112 (size=12001) 2024-11-21T00:26:59,762 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/19d2677ac9b243f0bad4ea3bac9ec4a9 2024-11-21T00:26:59,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e6e3254485ff4aa1bbd27cc7a79da1d5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5 2024-11-21T00:26:59,780 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T00:26:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/75a91be9321f4f72b6c516af3b4bcf9d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/75a91be9321f4f72b6c516af3b4bcf9d 2024-11-21T00:26:59,799 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/75a91be9321f4f72b6c516af3b4bcf9d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T00:26:59,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/19d2677ac9b243f0bad4ea3bac9ec4a9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/19d2677ac9b243f0bad4ea3bac9ec4a9 2024-11-21T00:26:59,818 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/19d2677ac9b243f0bad4ea3bac9ec4a9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T00:26:59,821 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d3834c10900814f7a41633aca86edf7e in 812ms, sequenceid=40, compaction requested=false 2024-11-21T00:26:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:26:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:26:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-21T00:26:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-21T00:26:59,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-21T00:26:59,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-21T00:26:59,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.4850 sec 2024-11-21T00:26:59,953 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:26:59,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:26:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:26:59,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215aa68e11a67e47179b2ae042e283ef8f_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148880012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148880019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148880022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148880024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148880025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741937_1113 (size=12154) 2024-11-21T00:27:00,053 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:00,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215aa68e11a67e47179b2ae042e283ef8f_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215aa68e11a67e47179b2ae042e283ef8f_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:00,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/373b7fc5cb244ce69503cfd5114aac2f, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:00,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/373b7fc5cb244ce69503cfd5114aac2f is 175, key is test_row_0/A:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741938_1114 (size=30955) 2024-11-21T00:27:00,108 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/373b7fc5cb244ce69503cfd5114aac2f 2024-11-21T00:27:00,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/c655eaddf7b44d0e851ccfd472253f4a is 50, key is test_row_0/B:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741939_1115 (size=12001) 2024-11-21T00:27:00,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148880145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148880146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148880146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148880146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148880146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148880362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148880363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148880363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148880364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148880364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-21T00:27:00,533 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:00,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-21T00:27:00,538 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:00,539 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:00,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T00:27:00,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/c655eaddf7b44d0e851ccfd472253f4a 2024-11-21T00:27:00,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b8a0099cc3724787a9e1aa46ab5d840b is 50, key is test_row_0/C:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741940_1116 (size=12001) 2024-11-21T00:27:00,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b8a0099cc3724787a9e1aa46ab5d840b 2024-11-21T00:27:00,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/373b7fc5cb244ce69503cfd5114aac2f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f 2024-11-21T00:27:00,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f, entries=150, sequenceid=57, filesize=30.2 K 2024-11-21T00:27:00,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/c655eaddf7b44d0e851ccfd472253f4a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/c655eaddf7b44d0e851ccfd472253f4a 2024-11-21T00:27:00,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/c655eaddf7b44d0e851ccfd472253f4a, entries=150, sequenceid=57, filesize=11.7 K 2024-11-21T00:27:00,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b8a0099cc3724787a9e1aa46ab5d840b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b8a0099cc3724787a9e1aa46ab5d840b 2024-11-21T00:27:00,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b8a0099cc3724787a9e1aa46ab5d840b, entries=150, sequenceid=57, filesize=11.7 K 2024-11-21T00:27:00,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for d3834c10900814f7a41633aca86edf7e in 686ms, sequenceid=57, compaction requested=true 2024-11-21T00:27:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:00,640 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:00,640 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:00,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:00,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:00,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:00,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:00,641 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:00,642 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:00,642 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,642 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/362f46cef6164b958a005dd0dc0b3d25, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/75a91be9321f4f72b6c516af3b4bcf9d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/c655eaddf7b44d0e851ccfd472253f4a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=35.2 K 2024-11-21T00:27:00,642 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:00,642 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:00,642 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,642 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=99.1 K 2024-11-21T00:27:00,642 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,643 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f] 2024-11-21T00:27:00,643 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 362f46cef6164b958a005dd0dc0b3d25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732148818461 2024-11-21T00:27:00,644 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ea47706f88417f897ac03e7050dd52, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732148818461 2024-11-21T00:27:00,644 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a91be9321f4f72b6c516af3b4bcf9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732148818512 2024-11-21T00:27:00,644 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e3254485ff4aa1bbd27cc7a79da1d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732148818512 2024-11-21T00:27:00,644 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c655eaddf7b44d0e851ccfd472253f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732148819290 2024-11-21T00:27:00,645 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 373b7fc5cb244ce69503cfd5114aac2f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732148819290 2024-11-21T00:27:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T00:27:00,669 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:00,678 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:00,679 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:00,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T00:27:00,679 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/f4c102eeaa2045c3bc7743d8981013b9 is 50, key is test_row_0/B:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:00,685 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121e2dc8c3bed024c6597a45d14fa84ae23_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:00,695 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121e2dc8c3bed024c6597a45d14fa84ae23_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:00,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,696 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2dc8c3bed024c6597a45d14fa84ae23_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:00,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741941_1117 (size=12104) 2024-11-21T00:27:00,719 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/f4c102eeaa2045c3bc7743d8981013b9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f4c102eeaa2045c3bc7743d8981013b9 2024-11-21T00:27:00,736 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into f4c102eeaa2045c3bc7743d8981013b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:00,737 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:00,737 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=13, startTime=1732148820640; duration=0sec 2024-11-21T00:27:00,737 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:00,737 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:00,737 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:00,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148880719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,740 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148880723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,740 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:00,740 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,740 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b1e448237c3b4b36aa9b09cbeb4c3aa5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/19d2677ac9b243f0bad4ea3bac9ec4a9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b8a0099cc3724787a9e1aa46ab5d840b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=35.2 K 2024-11-21T00:27:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148880726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148880731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,745 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e448237c3b4b36aa9b09cbeb4c3aa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732148818461 2024-11-21T00:27:00,746 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d2677ac9b243f0bad4ea3bac9ec4a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732148818512 2024-11-21T00:27:00,747 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8a0099cc3724787a9e1aa46ab5d840b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732148819290 2024-11-21T00:27:00,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148880739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217ede51daf0be4a63aaebb53b58c6894a_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148820008/Put/seqid=0 2024-11-21T00:27:00,775 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:27:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741942_1118 (size=4469) 2024-11-21T00:27:00,790 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:00,791 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/03107a4194764aba8fcdab7b93bf8c5b is 50, key is test_row_0/C:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,815 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#98 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:00,816 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/cfa245005c2b44fc89a119c485ebdbf1 is 175, key is test_row_0/A:col10/1732148819946/Put/seqid=0 2024-11-21T00:27:00,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741944_1120 (size=12104) 2024-11-21T00:27:00,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148880842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:00,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:00,854 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/03107a4194764aba8fcdab7b93bf8c5b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/03107a4194764aba8fcdab7b93bf8c5b 2024-11-21T00:27:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148880853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148880853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T00:27:00,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148880853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,865 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into 03107a4194764aba8fcdab7b93bf8c5b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:00,865 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:00,865 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=13, startTime=1732148820641; duration=0sec 2024-11-21T00:27:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,865 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,865 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741943_1119 (size=14594) 2024-11-21T00:27:00,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148880863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:00,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741945_1121 (size=31058) 2024-11-21T00:27:00,922 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/cfa245005c2b44fc89a119c485ebdbf1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/cfa245005c2b44fc89a119c485ebdbf1 2024-11-21T00:27:00,936 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into cfa245005c2b44fc89a119c485ebdbf1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:00,936 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:00,937 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=13, startTime=1732148820640; duration=0sec 2024-11-21T00:27:00,937 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:00,937 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:01,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148881052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148881069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148881072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148881073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148881082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T00:27:01,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:01,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217ede51daf0be4a63aaebb53b58c6894a_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ede51daf0be4a63aaebb53b58c6894a_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:01,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/3ba935e08c5442fea2c97410375d50be, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:01,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/3ba935e08c5442fea2c97410375d50be is 175, key is test_row_0/A:col10/1732148820008/Put/seqid=0 2024-11-21T00:27:01,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741946_1122 (size=39549) 2024-11-21T00:27:01,294 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/3ba935e08c5442fea2c97410375d50be 2024-11-21T00:27:01,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/710983c05dea42d39795ca84b9789041 is 50, key is test_row_0/B:col10/1732148820008/Put/seqid=0 2024-11-21T00:27:01,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741947_1123 (size=12001) 2024-11-21T00:27:01,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/710983c05dea42d39795ca84b9789041 2024-11-21T00:27:01,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148881372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148881382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148881382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148881393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/2189378c707449ad95a4e32c764afaca is 50, key is test_row_0/C:col10/1732148820008/Put/seqid=0 2024-11-21T00:27:01,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148881392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741948_1124 (size=12001) 2024-11-21T00:27:01,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/2189378c707449ad95a4e32c764afaca 2024-11-21T00:27:01,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/3ba935e08c5442fea2c97410375d50be as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be 2024-11-21T00:27:01,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:01,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be, entries=200, sequenceid=78, filesize=38.6 K 2024-11-21T00:27:01,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/710983c05dea42d39795ca84b9789041 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/710983c05dea42d39795ca84b9789041 2024-11-21T00:27:01,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/710983c05dea42d39795ca84b9789041, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T00:27:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/2189378c707449ad95a4e32c764afaca as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/2189378c707449ad95a4e32c764afaca 2024-11-21T00:27:01,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/2189378c707449ad95a4e32c764afaca, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T00:27:01,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d3834c10900814f7a41633aca86edf7e in 828ms, sequenceid=78, compaction requested=false 2024-11-21T00:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:01,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T00:27:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:01,625 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T00:27:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121569ddaecdd244ed3853ab6d390b16898_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148820722/Put/seqid=0 2024-11-21T00:27:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T00:27:01,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741949_1125 (size=12154) 2024-11-21T00:27:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:01,711 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121569ddaecdd244ed3853ab6d390b16898_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121569ddaecdd244ed3853ab6d390b16898_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/7d64140aefec46febf13b00b7234b8ec, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:01,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/7d64140aefec46febf13b00b7234b8ec is 175, key is test_row_0/A:col10/1732148820722/Put/seqid=0 2024-11-21T00:27:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741950_1126 (size=30955) 2024-11-21T00:27:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:01,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:01,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148881921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148881926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148881927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148881928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148881931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148882042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148882043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148882042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148882043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148882043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,145 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/7d64140aefec46febf13b00b7234b8ec 2024-11-21T00:27:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/83993c04962a48c099e50ec887d8809a is 50, key is test_row_0/B:col10/1732148820722/Put/seqid=0 2024-11-21T00:27:02,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741951_1127 (size=12001) 2024-11-21T00:27:02,222 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/83993c04962a48c099e50ec887d8809a 2024-11-21T00:27:02,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/c77c7f4356af463e93625b0d9f60ddb7 is 50, key is test_row_0/C:col10/1732148820722/Put/seqid=0 2024-11-21T00:27:02,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148882252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148882253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148882268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148882269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148882269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741952_1128 (size=12001) 2024-11-21T00:27:02,298 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/c77c7f4356af463e93625b0d9f60ddb7 2024-11-21T00:27:02,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/7d64140aefec46febf13b00b7234b8ec as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec 2024-11-21T00:27:02,312 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec, entries=150, sequenceid=96, filesize=30.2 K 2024-11-21T00:27:02,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/83993c04962a48c099e50ec887d8809a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/83993c04962a48c099e50ec887d8809a 2024-11-21T00:27:02,321 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/83993c04962a48c099e50ec887d8809a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-21T00:27:02,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/c77c7f4356af463e93625b0d9f60ddb7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c77c7f4356af463e93625b0d9f60ddb7 2024-11-21T00:27:02,347 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c77c7f4356af463e93625b0d9f60ddb7, entries=150, sequenceid=96, filesize=11.7 K 2024-11-21T00:27:02,349 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d3834c10900814f7a41633aca86edf7e in 724ms, sequenceid=96, compaction requested=true 2024-11-21T00:27:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-21T00:27:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-21T00:27:02,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-21T00:27:02,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-21T00:27:02,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.8200 sec 2024-11-21T00:27:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:02,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:02,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:27:02,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121800e3562352d4e719aeeaa248ccebd50_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148882601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148882610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148882615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148882617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148882621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741953_1129 (size=14594) 2024-11-21T00:27:02,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:02,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121800e3562352d4e719aeeaa248ccebd50_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121800e3562352d4e719aeeaa248ccebd50_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:02,664 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fa4afba19b1342a4b224deb1182f29b6, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:02,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fa4afba19b1342a4b224deb1182f29b6 is 175, key is test_row_0/A:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:02,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741954_1130 (size=39549) 2024-11-21T00:27:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T00:27:02,694 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:02,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T00:27:02,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:02,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:02,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:02,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148882724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148882725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148882733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148882733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148882733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:02,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:02,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:02,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:02,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148882944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148882945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148882953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148882955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148882942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:03,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fa4afba19b1342a4b224deb1182f29b6 2024-11-21T00:27:03,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/11b880494d4c450ba3df28b7026175b8 is 50, key is test_row_0/B:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:03,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741955_1131 (size=12001) 2024-11-21T00:27:03,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/11b880494d4c450ba3df28b7026175b8 2024-11-21T00:27:03,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/fc9b931fae2e4db587335953c39183c3 is 50, key is test_row_0/C:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:03,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741956_1132 (size=12001) 2024-11-21T00:27:03,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/fc9b931fae2e4db587335953c39183c3 2024-11-21T00:27:03,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fa4afba19b1342a4b224deb1182f29b6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6 2024-11-21T00:27:03,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6, entries=200, sequenceid=118, filesize=38.6 K 2024-11-21T00:27:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/11b880494d4c450ba3df28b7026175b8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/11b880494d4c450ba3df28b7026175b8 2024-11-21T00:27:03,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/11b880494d4c450ba3df28b7026175b8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:27:03,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/fc9b931fae2e4db587335953c39183c3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/fc9b931fae2e4db587335953c39183c3 2024-11-21T00:27:03,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/fc9b931fae2e4db587335953c39183c3, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:27:03,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d3834c10900814f7a41633aca86edf7e in 672ms, sequenceid=118, compaction requested=true 2024-11-21T00:27:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:03,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:03,239 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:03,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:03,239 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:03,241 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:03,241 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:03,241 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:03,241 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:03,241 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,241 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/cfa245005c2b44fc89a119c485ebdbf1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=137.8 K 2024-11-21T00:27:03,241 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,241 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/cfa245005c2b44fc89a119c485ebdbf1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6] 2024-11-21T00:27:03,241 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,242 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f4c102eeaa2045c3bc7743d8981013b9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/710983c05dea42d39795ca84b9789041, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/83993c04962a48c099e50ec887d8809a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/11b880494d4c450ba3df28b7026175b8] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=47.0 K 2024-11-21T00:27:03,242 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfa245005c2b44fc89a119c485ebdbf1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732148819290 2024-11-21T00:27:03,242 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c102eeaa2045c3bc7743d8981013b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732148819290 2024-11-21T00:27:03,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:03,242 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba935e08c5442fea2c97410375d50be, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732148820008 2024-11-21T00:27:03,242 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 710983c05dea42d39795ca84b9789041, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732148820008 2024-11-21T00:27:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:03,244 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 83993c04962a48c099e50ec887d8809a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732148820722 2024-11-21T00:27:03,244 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d64140aefec46febf13b00b7234b8ec, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732148820722 2024-11-21T00:27:03,244 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b880494d4c450ba3df28b7026175b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148821927 2024-11-21T00:27:03,245 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa4afba19b1342a4b224deb1182f29b6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148821925 2024-11-21T00:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:03,281 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:03,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:03,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T00:27:03,289 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b3ff6257a8f549978b12dcd651aa2845_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:03,291 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:03,292 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/f1c18cf63a5945f78e9cba8ea4e2edf6 is 50, key is test_row_0/B:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:03,294 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b3ff6257a8f549978b12dcd651aa2845_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:03,294 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b3ff6257a8f549978b12dcd651aa2845_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:03,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741957_1133 (size=12241) 2024-11-21T00:27:03,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741958_1134 (size=4469) 2024-11-21T00:27:03,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121af6c0df9bf0245438b1501e0f43cffde_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148823282/Put/seqid=0 2024-11-21T00:27:03,344 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#110 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:03,345 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/a6af7586e0d14d928d3fa52f4f041099 is 175, key is test_row_0/A:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:03,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:03,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148883367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148883370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148883372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148883372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148883375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741959_1135 (size=14794) 2024-11-21T00:27:03,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741960_1136 (size=31195) 2024-11-21T00:27:03,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:03,425 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121af6c0df9bf0245438b1501e0f43cffde_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121af6c0df9bf0245438b1501e0f43cffde_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:03,427 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/594d470bca9449be9d0e2dc6eff8847e, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:03,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/594d470bca9449be9d0e2dc6eff8847e is 175, key is test_row_0/A:col10/1732148823282/Put/seqid=0 2024-11-21T00:27:03,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741961_1137 (size=39749) 2024-11-21T00:27:03,473 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/594d470bca9449be9d0e2dc6eff8847e 2024-11-21T00:27:03,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148883482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148883482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/6a172523da514e51b958f8e3afe491bf is 50, key is test_row_0/B:col10/1732148823282/Put/seqid=0 2024-11-21T00:27:03,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148883493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148883493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148883493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741962_1138 (size=12151) 2024-11-21T00:27:03,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/6a172523da514e51b958f8e3afe491bf 2024-11-21T00:27:03,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/7bf258c9a74444e7af37fd4df44cfce9 is 50, key is test_row_0/C:col10/1732148823282/Put/seqid=0 2024-11-21T00:27:03,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741963_1139 (size=12151) 2024-11-21T00:27:03,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148883692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148883692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148883712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148883712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148883712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:03,729 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/f1c18cf63a5945f78e9cba8ea4e2edf6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f1c18cf63a5945f78e9cba8ea4e2edf6 2024-11-21T00:27:03,742 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into f1c18cf63a5945f78e9cba8ea4e2edf6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:03,742 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:03,742 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=12, startTime=1732148823239; duration=0sec 2024-11-21T00:27:03,743 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:03,743 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:03,743 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:03,746 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:03,746 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:03,746 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,746 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/03107a4194764aba8fcdab7b93bf8c5b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/2189378c707449ad95a4e32c764afaca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c77c7f4356af463e93625b0d9f60ddb7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/fc9b931fae2e4db587335953c39183c3] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=47.0 K 2024-11-21T00:27:03,747 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 03107a4194764aba8fcdab7b93bf8c5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732148819290 2024-11-21T00:27:03,747 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2189378c707449ad95a4e32c764afaca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732148820008 2024-11-21T00:27:03,748 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c77c7f4356af463e93625b0d9f60ddb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732148820722 2024-11-21T00:27:03,748 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting fc9b931fae2e4db587335953c39183c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148821927 2024-11-21T00:27:03,787 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:03,788 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/97b81ce4874e4f8397aee10179f4a93b is 50, key is test_row_0/C:col10/1732148821928/Put/seqid=0 2024-11-21T00:27:03,804 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/a6af7586e0d14d928d3fa52f4f041099 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/a6af7586e0d14d928d3fa52f4f041099 2024-11-21T00:27:03,815 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into a6af7586e0d14d928d3fa52f4f041099(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:03,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,815 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:03,815 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=12, startTime=1732148823239; duration=0sec 2024-11-21T00:27:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,815 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:03,815 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741964_1140 (size=12241) 2024-11-21T00:27:03,837 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/97b81ce4874e4f8397aee10179f4a93b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/97b81ce4874e4f8397aee10179f4a93b 2024-11-21T00:27:03,843 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into 97b81ce4874e4f8397aee10179f4a93b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:03,843 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:03,843 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=12, startTime=1732148823243; duration=0sec 2024-11-21T00:27:03,843 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:03,843 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:03,888 INFO [master/5ed4808ef0e6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T00:27:03,888 INFO [master/5ed4808ef0e6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T00:27:03,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:03,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:03,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:03,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148884012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148884012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148884022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/7bf258c9a74444e7af37fd4df44cfce9 2024-11-21T00:27:04,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/594d470bca9449be9d0e2dc6eff8847e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e 2024-11-21T00:27:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148884033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148884033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e, entries=200, sequenceid=136, filesize=38.8 K 2024-11-21T00:27:04,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/6a172523da514e51b958f8e3afe491bf as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6a172523da514e51b958f8e3afe491bf 2024-11-21T00:27:04,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6a172523da514e51b958f8e3afe491bf, entries=150, sequenceid=136, filesize=11.9 K 2024-11-21T00:27:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/7bf258c9a74444e7af37fd4df44cfce9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/7bf258c9a74444e7af37fd4df44cfce9 2024-11-21T00:27:04,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/7bf258c9a74444e7af37fd4df44cfce9, entries=150, sequenceid=136, filesize=11.9 K 2024-11-21T00:27:04,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d3834c10900814f7a41633aca86edf7e in 776ms, sequenceid=136, compaction requested=false 2024-11-21T00:27:04,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:04,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T00:27:04,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:04,122 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:27:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112153edade836e5439188c9f645b5731521_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148823359/Put/seqid=0 2024-11-21T00:27:04,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741965_1141 (size=12304) 2024-11-21T00:27:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:04,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148884550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148884553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148884553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148884558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148884559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:04,603 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112153edade836e5439188c9f645b5731521_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153edade836e5439188c9f645b5731521_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c45bebeccc54412ba39d7f33d92765de, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c45bebeccc54412ba39d7f33d92765de is 175, key is test_row_0/A:col10/1732148823359/Put/seqid=0 2024-11-21T00:27:04,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741966_1142 (size=31105) 2024-11-21T00:27:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148884662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148884665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148884665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148884667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148884666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148884873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148884873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148884873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148884874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148884873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,038 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c45bebeccc54412ba39d7f33d92765de 2024-11-21T00:27:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3792c8f0211444e0a8a9c9002fa32ff8 is 50, key is test_row_0/B:col10/1732148823359/Put/seqid=0 2024-11-21T00:27:05,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741967_1143 (size=12151) 2024-11-21T00:27:05,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148885192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148885193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148885193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148885193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148885193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,532 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3792c8f0211444e0a8a9c9002fa32ff8 2024-11-21T00:27:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/440d392cef8b4d518127ec5f797f120b is 50, key is test_row_0/C:col10/1732148823359/Put/seqid=0 2024-11-21T00:27:05,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741968_1144 (size=12151) 2024-11-21T00:27:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148885712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148885713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148885713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148885713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148885713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,014 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/440d392cef8b4d518127ec5f797f120b 2024-11-21T00:27:06,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c45bebeccc54412ba39d7f33d92765de as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de 2024-11-21T00:27:06,031 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de, entries=150, sequenceid=157, filesize=30.4 K 2024-11-21T00:27:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3792c8f0211444e0a8a9c9002fa32ff8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3792c8f0211444e0a8a9c9002fa32ff8 2024-11-21T00:27:06,038 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3792c8f0211444e0a8a9c9002fa32ff8, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T00:27:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/440d392cef8b4d518127ec5f797f120b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/440d392cef8b4d518127ec5f797f120b 2024-11-21T00:27:06,044 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/440d392cef8b4d518127ec5f797f120b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T00:27:06,045 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d3834c10900814f7a41633aca86edf7e in 1923ms, sequenceid=157, compaction requested=true 2024-11-21T00:27:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T00:27:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T00:27:06,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T00:27:06,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3460 sec 2024-11-21T00:27:06,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.3610 sec 2024-11-21T00:27:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:06,727 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:06,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:27:06,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c22dc84ea6ee4fdc9a86bd725f054514_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148886769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148886770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148886771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148886773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148886775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741969_1145 (size=14794) 2024-11-21T00:27:06,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:06,835 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c22dc84ea6ee4fdc9a86bd725f054514_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c22dc84ea6ee4fdc9a86bd725f054514_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:06,836 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/392313474d2e4777924512d6b0f2baec, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:06,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/392313474d2e4777924512d6b0f2baec is 175, key is test_row_0/A:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T00:27:06,873 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:06,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T00:27:06,876 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:06,879 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:06,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:27:06,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148886893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148886893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148886894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741970_1146 (size=39749) 2024-11-21T00:27:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148886894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/392313474d2e4777924512d6b0f2baec 2024-11-21T00:27:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148886894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:06,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/d85801383e5044e3be8589816919f84b is 50, key is test_row_0/B:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:27:07,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741971_1147 (size=12151) 2024-11-21T00:27:07,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/d85801383e5044e3be8589816919f84b 2024-11-21T00:27:07,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/ec9f69facf9d417e9c1fd48262880a2f is 50, key is test_row_0/C:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:07,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741972_1148 (size=12151) 2024-11-21T00:27:07,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/ec9f69facf9d417e9c1fd48262880a2f 2024-11-21T00:27:07,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/392313474d2e4777924512d6b0f2baec as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec 2024-11-21T00:27:07,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec, entries=200, sequenceid=176, filesize=38.8 K 2024-11-21T00:27:07,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/d85801383e5044e3be8589816919f84b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/d85801383e5044e3be8589816919f84b 2024-11-21T00:27:07,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/d85801383e5044e3be8589816919f84b, entries=150, sequenceid=176, filesize=11.9 K 2024-11-21T00:27:07,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/ec9f69facf9d417e9c1fd48262880a2f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/ec9f69facf9d417e9c1fd48262880a2f 2024-11-21T00:27:07,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/ec9f69facf9d417e9c1fd48262880a2f, entries=150, sequenceid=176, filesize=11.9 K 2024-11-21T00:27:07,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d3834c10900814f7a41633aca86edf7e in 361ms, sequenceid=176, compaction requested=true 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:07,089 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:07,089 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:07,091 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:07,091 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:07,091 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:07,091 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:07,091 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,091 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,091 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f1c18cf63a5945f78e9cba8ea4e2edf6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6a172523da514e51b958f8e3afe491bf, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3792c8f0211444e0a8a9c9002fa32ff8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/d85801383e5044e3be8589816919f84b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=47.6 K 2024-11-21T00:27:07,091 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/a6af7586e0d14d928d3fa52f4f041099, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=138.5 K 2024-11-21T00:27:07,091 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,091 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/a6af7586e0d14d928d3fa52f4f041099, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec] 2024-11-21T00:27:07,092 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6af7586e0d14d928d3fa52f4f041099, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148821927 2024-11-21T00:27:07,092 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 594d470bca9449be9d0e2dc6eff8847e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732148822614 2024-11-21T00:27:07,092 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c18cf63a5945f78e9cba8ea4e2edf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148821927 2024-11-21T00:27:07,093 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45bebeccc54412ba39d7f33d92765de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732148823359 2024-11-21T00:27:07,093 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 392313474d2e4777924512d6b0f2baec, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732148824547 2024-11-21T00:27:07,094 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a172523da514e51b958f8e3afe491bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732148823279 2024-11-21T00:27:07,095 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3792c8f0211444e0a8a9c9002fa32ff8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732148823359 2024-11-21T00:27:07,096 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d85801383e5044e3be8589816919f84b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732148824547 2024-11-21T00:27:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:07,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:07,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:27:07,117 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:07,130 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:07,131 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3312135ac6d64da5b0c3bd4f12620bfb is 50, key is test_row_0/B:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:07,133 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f52d3fc7162d4222b3ff15d63e69f984_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:07,139 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f52d3fc7162d4222b3ff15d63e69f984_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:07,139 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f52d3fc7162d4222b3ff15d63e69f984_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:07,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126cbcdc8a9a04995b66479db5b480d97_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148826773/Put/seqid=0 2024-11-21T00:27:07,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148887154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148887155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148887156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148887158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148887171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741974_1150 (size=4469) 2024-11-21T00:27:07,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741973_1149 (size=12527) 2024-11-21T00:27:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:27:07,209 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3312135ac6d64da5b0c3bd4f12620bfb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3312135ac6d64da5b0c3bd4f12620bfb 2024-11-21T00:27:07,217 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into 3312135ac6d64da5b0c3bd4f12620bfb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:07,217 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:07,217 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=12, startTime=1732148827089; duration=0sec 2024-11-21T00:27:07,218 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:07,218 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:07,218 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:07,219 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:07,220 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:07,220 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,220 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/97b81ce4874e4f8397aee10179f4a93b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/7bf258c9a74444e7af37fd4df44cfce9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/440d392cef8b4d518127ec5f797f120b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/ec9f69facf9d417e9c1fd48262880a2f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=47.6 K 2024-11-21T00:27:07,220 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b81ce4874e4f8397aee10179f4a93b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148821927 2024-11-21T00:27:07,221 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf258c9a74444e7af37fd4df44cfce9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732148823279 2024-11-21T00:27:07,221 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 440d392cef8b4d518127ec5f797f120b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732148823359 2024-11-21T00:27:07,222 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9f69facf9d417e9c1fd48262880a2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732148824547 2024-11-21T00:27:07,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741975_1151 (size=12304) 2024-11-21T00:27:07,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:07,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112126cbcdc8a9a04995b66479db5b480d97_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126cbcdc8a9a04995b66479db5b480d97_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:07,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/b27967ec3006495ebefa6e4be2b51f19, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:07,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/b27967ec3006495ebefa6e4be2b51f19 is 175, key is test_row_0/A:col10/1732148826773/Put/seqid=0 2024-11-21T00:27:07,249 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:07,250 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/bf530600369146f7a376956846aa4464 is 50, key is test_row_0/C:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:07,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741976_1152 (size=31105) 2024-11-21T00:27:07,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/b27967ec3006495ebefa6e4be2b51f19 2024-11-21T00:27:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148887274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148887284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148887283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/90eb60a68a8e494aaf50bb35bdc38eec is 50, key is test_row_0/B:col10/1732148826773/Put/seqid=0 2024-11-21T00:27:07,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741977_1153 (size=12527) 2024-11-21T00:27:07,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:07,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741978_1154 (size=12151) 2024-11-21T00:27:07,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/90eb60a68a8e494aaf50bb35bdc38eec 2024-11-21T00:27:07,417 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/bf530600369146f7a376956846aa4464 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bf530600369146f7a376956846aa4464 2024-11-21T00:27:07,424 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into bf530600369146f7a376956846aa4464(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:07,424 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:07,424 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=12, startTime=1732148827089; duration=0sec 2024-11-21T00:27:07,424 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:07,424 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:07,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/bb24edc6d4d741e6a051f6405933a562 is 50, key is test_row_0/C:col10/1732148826773/Put/seqid=0 2024-11-21T00:27:07,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741979_1155 (size=12151) 2024-11-21T00:27:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148887482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148887483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148887483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:27:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148887533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148887533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,589 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#122 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:07,590 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/67cbfc0624d545e19616c30f106b6581 is 175, key is test_row_0/A:col10/1732148824547/Put/seqid=0 2024-11-21T00:27:07,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741980_1156 (size=31481) 2024-11-21T00:27:07,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148887792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148887793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148887793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:07,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:07,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:07,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:07,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/bb24edc6d4d741e6a051f6405933a562 2024-11-21T00:27:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148887876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148887892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:07,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:07,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:07,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/b27967ec3006495ebefa6e4be2b51f19 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19 2024-11-21T00:27:07,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19, entries=150, sequenceid=195, filesize=30.4 K 2024-11-21T00:27:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/90eb60a68a8e494aaf50bb35bdc38eec as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/90eb60a68a8e494aaf50bb35bdc38eec 2024-11-21T00:27:07,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/90eb60a68a8e494aaf50bb35bdc38eec, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T00:27:07,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/bb24edc6d4d741e6a051f6405933a562 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bb24edc6d4d741e6a051f6405933a562 2024-11-21T00:27:07,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bb24edc6d4d741e6a051f6405933a562, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T00:27:07,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d3834c10900814f7a41633aca86edf7e in 839ms, sequenceid=195, compaction requested=false 2024-11-21T00:27:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:08,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T00:27:08,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:08,013 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:27:08,022 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/67cbfc0624d545e19616c30f106b6581 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67cbfc0624d545e19616c30f106b6581 2024-11-21T00:27:08,033 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into 67cbfc0624d545e19616c30f106b6581(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:08,033 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:08,033 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=12, startTime=1732148827089; duration=0sec 2024-11-21T00:27:08,033 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:08,033 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:27:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e8ee12d436045c5b3a8c505865e26fb_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148827151/Put/seqid=0 2024-11-21T00:27:08,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741981_1157 (size=12304) 2024-11-21T00:27:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:08,075 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e8ee12d436045c5b3a8c505865e26fb_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e8ee12d436045c5b3a8c505865e26fb_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:08,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e1c42c7dc34843db90c9b7dc5b405b18, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:08,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e1c42c7dc34843db90c9b7dc5b405b18 is 175, key is test_row_0/A:col10/1732148827151/Put/seqid=0 2024-11-21T00:27:08,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741982_1158 (size=31105) 2024-11-21T00:27:08,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148888361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148888373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148888374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148888394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148888405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148888488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148888488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148888489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,521 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e1c42c7dc34843db90c9b7dc5b405b18 2024-11-21T00:27:08,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/88dc45c3962b475f807c0435ed372a87 is 50, key is test_row_0/B:col10/1732148827151/Put/seqid=0 2024-11-21T00:27:08,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741983_1159 (size=12151) 2024-11-21T00:27:08,582 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/88dc45c3962b475f807c0435ed372a87 2024-11-21T00:27:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/de095100656d4a0699467b8e3948a8e8 is 50, key is test_row_0/C:col10/1732148827151/Put/seqid=0 2024-11-21T00:27:08,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741984_1160 (size=12151) 2024-11-21T00:27:08,628 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/de095100656d4a0699467b8e3948a8e8 2024-11-21T00:27:08,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/e1c42c7dc34843db90c9b7dc5b405b18 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18 2024-11-21T00:27:08,640 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18, entries=150, sequenceid=214, filesize=30.4 K 2024-11-21T00:27:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/88dc45c3962b475f807c0435ed372a87 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/88dc45c3962b475f807c0435ed372a87 2024-11-21T00:27:08,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/88dc45c3962b475f807c0435ed372a87, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T00:27:08,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/de095100656d4a0699467b8e3948a8e8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/de095100656d4a0699467b8e3948a8e8 2024-11-21T00:27:08,669 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/de095100656d4a0699467b8e3948a8e8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T00:27:08,672 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for d3834c10900814f7a41633aca86edf7e in 659ms, sequenceid=214, compaction requested=true 2024-11-21T00:27:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T00:27:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T00:27:08,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T00:27:08,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-21T00:27:08,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8140 sec 2024-11-21T00:27:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:08,708 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:08,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:27:08,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112107c9470435884daca6e3edf15e1c07a6_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741985_1161 (size=19774) 2024-11-21T00:27:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148888780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148888787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148888800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148888894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148888914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:08,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148888914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T00:27:09,052 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:09,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T00:27:09,055 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:09,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:09,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:27:09,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148889102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148889124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148889124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:27:09,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:09,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112107c9470435884daca6e3edf15e1c07a6_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112107c9470435884daca6e3edf15e1c07a6_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:09,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9b558da147449408c40359140414556, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:09,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9b558da147449408c40359140414556 is 175, key is test_row_0/A:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:09,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741986_1162 (size=57033) 2024-11-21T00:27:09,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9b558da147449408c40359140414556 2024-11-21T00:27:09,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3384fafc042a4b349db82c0d96c46178 is 50, key is test_row_0/B:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:09,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741987_1163 (size=12151) 2024-11-21T00:27:09,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3384fafc042a4b349db82c0d96c46178 2024-11-21T00:27:09,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/cd9f4c1a40e54d459eff7853d27aa928 is 50, key is test_row_0/C:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:09,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:27:09,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741988_1164 (size=12151) 2024-11-21T00:27:09,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/cd9f4c1a40e54d459eff7853d27aa928 2024-11-21T00:27:09,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9b558da147449408c40359140414556 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556 2024-11-21T00:27:09,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556, entries=300, sequenceid=238, filesize=55.7 K 2024-11-21T00:27:09,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/3384fafc042a4b349db82c0d96c46178 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3384fafc042a4b349db82c0d96c46178 2024-11-21T00:27:09,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3384fafc042a4b349db82c0d96c46178, entries=150, sequenceid=238, filesize=11.9 K 2024-11-21T00:27:09,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/cd9f4c1a40e54d459eff7853d27aa928 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd9f4c1a40e54d459eff7853d27aa928 2024-11-21T00:27:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148889413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148889422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,427 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in d3834c10900814f7a41633aca86edf7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd9f4c1a40e54d459eff7853d27aa928, entries=150, sequenceid=238, filesize=11.9 K 2024-11-21T00:27:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148889422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d3834c10900814f7a41633aca86edf7e in 724ms, sequenceid=238, compaction requested=true 2024-11-21T00:27:09,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:09,430 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in d3834c10900814f7a41633aca86edf7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:09,434 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:27:09,434 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150724 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:09,436 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,436 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67cbfc0624d545e19616c30f106b6581, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=147.2 K 2024-11-21T00:27:09,436 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67cbfc0624d545e19616c30f106b6581, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556] 2024-11-21T00:27:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:09,436 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,436 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3312135ac6d64da5b0c3bd4f12620bfb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/90eb60a68a8e494aaf50bb35bdc38eec, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/88dc45c3962b475f807c0435ed372a87, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3384fafc042a4b349db82c0d96c46178] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=47.8 K 2024-11-21T00:27:09,437 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 67cbfc0624d545e19616c30f106b6581, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732148824547 2024-11-21T00:27:09,437 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3312135ac6d64da5b0c3bd4f12620bfb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732148824547 2024-11-21T00:27:09,437 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b27967ec3006495ebefa6e4be2b51f19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732148826773 2024-11-21T00:27:09,437 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90eb60a68a8e494aaf50bb35bdc38eec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732148826773 2024-11-21T00:27:09,438 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88dc45c3962b475f807c0435ed372a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732148827151 2024-11-21T00:27:09,438 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e1c42c7dc34843db90c9b7dc5b405b18, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732148827151 2024-11-21T00:27:09,438 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b558da147449408c40359140414556, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732148828348 2024-11-21T00:27:09,439 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3384fafc042a4b349db82c0d96c46178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732148828348 2024-11-21T00:27:09,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:09,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T00:27:09,502 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:09,512 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#135 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:09,512 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/6f48cd8eb08d42ddb9bc427c82238dc7 is 50, key is test_row_0/B:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:09,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:09,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dbb23f4c8bbd4a0fa5695dd9ff12e7eb_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148828777/Put/seqid=0 2024-11-21T00:27:09,537 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121bdf7cb39ee4f4814b236386e37e644ce_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:09,540 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121bdf7cb39ee4f4814b236386e37e644ce_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:09,540 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bdf7cb39ee4f4814b236386e37e644ce_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:09,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148889568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741989_1165 (size=12663) 2024-11-21T00:27:09,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741990_1166 (size=14794) 2024-11-21T00:27:09,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148889573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741991_1167 (size=4469) 2024-11-21T00:27:09,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:27:09,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148889683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148889685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:09,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:09,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:09,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148889895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148889908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148889933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:09,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:09,982 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dbb23f4c8bbd4a0fa5695dd9ff12e7eb_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dbb23f4c8bbd4a0fa5695dd9ff12e7eb_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:09,984 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/6f48cd8eb08d42ddb9bc427c82238dc7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6f48cd8eb08d42ddb9bc427c82238dc7 2024-11-21T00:27:09,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/5862a0f2a0de44efae4172d867af9bd1, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:09,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/5862a0f2a0de44efae4172d867af9bd1 is 175, key is test_row_0/A:col10/1732148828777/Put/seqid=0 2024-11-21T00:27:09,997 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into 6f48cd8eb08d42ddb9bc427c82238dc7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:09,997 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:09,998 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=12, startTime=1732148829434; duration=0sec 2024-11-21T00:27:09,998 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:09,998 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:09,998 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:09,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:10,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:10,006 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#134 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:10,006 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/b2ebe8b922194be8a2a848cbf919e52f is 175, key is test_row_0/A:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:10,006 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:10,006 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:10,006 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:10,007 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bf530600369146f7a376956846aa4464, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bb24edc6d4d741e6a051f6405933a562, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/de095100656d4a0699467b8e3948a8e8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd9f4c1a40e54d459eff7853d27aa928] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=47.8 K 2024-11-21T00:27:10,008 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf530600369146f7a376956846aa4464, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732148824547 2024-11-21T00:27:10,009 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb24edc6d4d741e6a051f6405933a562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732148826773 2024-11-21T00:27:10,011 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting de095100656d4a0699467b8e3948a8e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732148827151 2024-11-21T00:27:10,012 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd9f4c1a40e54d459eff7853d27aa928, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732148828348 2024-11-21T00:27:10,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741992_1168 (size=39749) 2024-11-21T00:27:10,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/5862a0f2a0de44efae4172d867af9bd1 2024-11-21T00:27:10,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/0250e751773f47b1b9c1a71edf201a87 is 50, key is test_row_0/B:col10/1732148828777/Put/seqid=0 2024-11-21T00:27:10,079 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:10,079 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/cd64bcbda5874419a49f3a3e071f473d is 50, key is test_row_0/C:col10/1732148828348/Put/seqid=0 2024-11-21T00:27:10,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741993_1169 (size=31617) 2024-11-21T00:27:10,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741994_1170 (size=12151) 2024-11-21T00:27:10,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/0250e751773f47b1b9c1a71edf201a87 2024-11-21T00:27:10,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:10,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:10,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:10,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:10,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:10,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:10,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0cf38d6f0a4342e79106ca23ec443e30 is 50, key is test_row_0/C:col10/1732148828777/Put/seqid=0 2024-11-21T00:27:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:27:10,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741995_1171 (size=12663) 2024-11-21T00:27:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148890217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148890237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741996_1172 (size=12151) 2024-11-21T00:27:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0cf38d6f0a4342e79106ca23ec443e30 2024-11-21T00:27:10,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/5862a0f2a0de44efae4172d867af9bd1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1 2024-11-21T00:27:10,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1, entries=200, sequenceid=253, filesize=38.8 K 2024-11-21T00:27:10,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/0250e751773f47b1b9c1a71edf201a87 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/0250e751773f47b1b9c1a71edf201a87 2024-11-21T00:27:10,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/0250e751773f47b1b9c1a71edf201a87, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T00:27:10,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0cf38d6f0a4342e79106ca23ec443e30 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0cf38d6f0a4342e79106ca23ec443e30 2024-11-21T00:27:10,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0cf38d6f0a4342e79106ca23ec443e30, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T00:27:10,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d3834c10900814f7a41633aca86edf7e in 861ms, sequenceid=253, compaction requested=false 2024-11-21T00:27:10,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:10,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T00:27:10,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:10,329 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T00:27:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bc94f7cf0ecc47339fee455da71f62fa_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148829570/Put/seqid=0 2024-11-21T00:27:10,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741997_1173 (size=12454) 2024-11-21T00:27:10,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:10,449 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bc94f7cf0ecc47339fee455da71f62fa_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bc94f7cf0ecc47339fee455da71f62fa_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:10,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0970417e741748c699f2557ca6859565, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0970417e741748c699f2557ca6859565 is 175, key is test_row_0/A:col10/1732148829570/Put/seqid=0 2024-11-21T00:27:10,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741998_1174 (size=31255) 2024-11-21T00:27:10,495 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0970417e741748c699f2557ca6859565 2024-11-21T00:27:10,508 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/b2ebe8b922194be8a2a848cbf919e52f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b2ebe8b922194be8a2a848cbf919e52f 2024-11-21T00:27:10,515 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into b2ebe8b922194be8a2a848cbf919e52f(size=30.9 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-21T00:27:10,516 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:10,516 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=12, startTime=1732148829434; duration=1sec 2024-11-21T00:27:10,516 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:10,516 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:10,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/bda3fa5e9e0f46b4845b0e8f952882c2 is 50, key is test_row_0/B:col10/1732148829570/Put/seqid=0 2024-11-21T00:27:10,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741999_1175 (size=12301) 2024-11-21T00:27:10,613 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/cd64bcbda5874419a49f3a3e071f473d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd64bcbda5874419a49f3a3e071f473d 2024-11-21T00:27:10,622 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into cd64bcbda5874419a49f3a3e071f473d(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:10,622 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:10,622 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=12, startTime=1732148829434; duration=0sec 2024-11-21T00:27:10,622 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:10,622 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:10,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148890804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148890804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148890912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148890912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148890953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:10,954 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in d3834c10900814f7a41633aca86edf7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:10,992 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/bda3fa5e9e0f46b4845b0e8f952882c2 2024-11-21T00:27:11,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d68a44f2acfc4ee992cea2f5ab014549 is 50, key is test_row_0/C:col10/1732148829570/Put/seqid=0 2024-11-21T00:27:11,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742000_1176 (size=12301) 2024-11-21T00:27:11,074 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d68a44f2acfc4ee992cea2f5ab014549 2024-11-21T00:27:11,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0970417e741748c699f2557ca6859565 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565 2024-11-21T00:27:11,108 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565, entries=150, sequenceid=275, filesize=30.5 K 2024-11-21T00:27:11,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/bda3fa5e9e0f46b4845b0e8f952882c2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/bda3fa5e9e0f46b4845b0e8f952882c2 2024-11-21T00:27:11,118 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/bda3fa5e9e0f46b4845b0e8f952882c2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T00:27:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d68a44f2acfc4ee992cea2f5ab014549 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d68a44f2acfc4ee992cea2f5ab014549 2024-11-21T00:27:11,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148891124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148891123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,140 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d68a44f2acfc4ee992cea2f5ab014549, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T00:27:11,141 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d3834c10900814f7a41633aca86edf7e in 812ms, sequenceid=275, compaction requested=true 2024-11-21T00:27:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T00:27:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T00:27:11,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T00:27:11,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-21T00:27:11,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0930 sec 2024-11-21T00:27:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T00:27:11,219 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:11,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T00:27:11,222 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:27:11,222 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:11,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:27:11,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T00:27:11,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:11,376 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T00:27:11,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212146587ea8d44994882580a6c996a135_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148830802/Put/seqid=0 2024-11-21T00:27:11,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:11,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742001_1177 (size=12454) 2024-11-21T00:27:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:11,471 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212146587ea8d44994882580a6c996a135_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212146587ea8d44994882580a6c996a135_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:11,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/83fb521292bb4300971fa263a68b3e37, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/83fb521292bb4300971fa263a68b3e37 is 175, key is test_row_0/A:col10/1732148830802/Put/seqid=0 2024-11-21T00:27:11,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742002_1178 (size=31255) 2024-11-21T00:27:11,495 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/83fb521292bb4300971fa263a68b3e37 2024-11-21T00:27:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148891497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/831125ce47dd444fb00cba81fa89c54c is 50, key is test_row_0/B:col10/1732148830802/Put/seqid=0 2024-11-21T00:27:11,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148891500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148891501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148891505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742003_1179 (size=12301) 2024-11-21T00:27:11,541 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/831125ce47dd444fb00cba81fa89c54c 2024-11-21T00:27:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:27:11,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0c0159ff993f4deb8e42d365cb99afbb is 50, key is test_row_0/C:col10/1732148830802/Put/seqid=0 2024-11-21T00:27:11,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742004_1180 (size=12301) 2024-11-21T00:27:11,598 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0c0159ff993f4deb8e42d365cb99afbb 2024-11-21T00:27:11,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/83fb521292bb4300971fa263a68b3e37 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37 2024-11-21T00:27:11,611 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37, entries=150, sequenceid=292, filesize=30.5 K 2024-11-21T00:27:11,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/831125ce47dd444fb00cba81fa89c54c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/831125ce47dd444fb00cba81fa89c54c 2024-11-21T00:27:11,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148891613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148891622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148891622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148891622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,627 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/831125ce47dd444fb00cba81fa89c54c, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T00:27:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0c0159ff993f4deb8e42d365cb99afbb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0c0159ff993f4deb8e42d365cb99afbb 2024-11-21T00:27:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,634 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0c0159ff993f4deb8e42d365cb99afbb, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T00:27:11,635 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d3834c10900814f7a41633aca86edf7e in 259ms, sequenceid=292, compaction requested=true 2024-11-21T00:27:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T00:27:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T00:27:11,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T00:27:11,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 415 msec 2024-11-21T00:27:11,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 430 msec 2024-11-21T00:27:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:11,825 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:11,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:27:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148891840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ab9d29deed441da91692f17a6c89dfd_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742005_1181 (size=14994) 2024-11-21T00:27:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T00:27:11,853 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:11,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:11,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148891841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T00:27:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148891843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:27:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148891844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:11,859 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:11,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:11,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ab9d29deed441da91692f17a6c89dfd_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ab9d29deed441da91692f17a6c89dfd_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:11,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/232f2c0f80e24a45a197b3ce8182ff9a, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:11,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/232f2c0f80e24a45a197b3ce8182ff9a is 175, key is test_row_0/A:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:11,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742006_1182 (size=39949) 2024-11-21T00:27:11,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/232f2c0f80e24a45a197b3ce8182ff9a 2024-11-21T00:27:11,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/171e7b0d616541869910243590a09a1a is 50, key is test_row_0/B:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148891952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:27:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742007_1183 (size=12301) 2024-11-21T00:27:11,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148891962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148891962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/171e7b0d616541869910243590a09a1a 2024-11-21T00:27:11,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148891975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:11,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0f1522f6cb0b4e9684421eb94abc2bfe is 50, key is test_row_0/C:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:12,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:27:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742008_1184 (size=12301) 2024-11-21T00:27:12,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148892166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:27:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:27:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148892182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148892184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148892184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:27:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0f1522f6cb0b4e9684421eb94abc2bfe 2024-11-21T00:27:12,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/232f2c0f80e24a45a197b3ce8182ff9a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a 2024-11-21T00:27:12,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:27:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:27:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:12,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a, entries=200, sequenceid=314, filesize=39.0 K 2024-11-21T00:27:12,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/171e7b0d616541869910243590a09a1a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/171e7b0d616541869910243590a09a1a 2024-11-21T00:27:12,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148892504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148892504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148892504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148892504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:12,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/171e7b0d616541869910243590a09a1a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-21T00:27:12,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/0f1522f6cb0b4e9684421eb94abc2bfe as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0f1522f6cb0b4e9684421eb94abc2bfe 2024-11-21T00:27:12,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0f1522f6cb0b4e9684421eb94abc2bfe, entries=150, sequenceid=314, filesize=12.0 K 2024-11-21T00:27:12,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d3834c10900814f7a41633aca86edf7e in 707ms, sequenceid=314, compaction requested=true 2024-11-21T00:27:12,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:12,533 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:27:12,533 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:27:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:12,535 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61717 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:27:12,535 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:12,535 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,536 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6f48cd8eb08d42ddb9bc427c82238dc7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/0250e751773f47b1b9c1a71edf201a87, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/bda3fa5e9e0f46b4845b0e8f952882c2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/831125ce47dd444fb00cba81fa89c54c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/171e7b0d616541869910243590a09a1a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=60.3 K 2024-11-21T00:27:12,536 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173825 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:27:12,536 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:12,536 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,536 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b2ebe8b922194be8a2a848cbf919e52f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=169.8 K 2024-11-21T00:27:12,536 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,536 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b2ebe8b922194be8a2a848cbf919e52f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a] 2024-11-21T00:27:12,537 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f48cd8eb08d42ddb9bc427c82238dc7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732148828348 2024-11-21T00:27:12,537 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ebe8b922194be8a2a848cbf919e52f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732148828348 2024-11-21T00:27:12,537 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0250e751773f47b1b9c1a71edf201a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732148828746 2024-11-21T00:27:12,538 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5862a0f2a0de44efae4172d867af9bd1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732148828746 2024-11-21T00:27:12,540 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bda3fa5e9e0f46b4845b0e8f952882c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148829529 2024-11-21T00:27:12,540 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0970417e741748c699f2557ca6859565, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148829529 2024-11-21T00:27:12,541 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 831125ce47dd444fb00cba81fa89c54c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148830769 2024-11-21T00:27:12,541 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83fb521292bb4300971fa263a68b3e37, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148830769 2024-11-21T00:27:12,542 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232f2c0f80e24a45a197b3ce8182ff9a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732148831490 2024-11-21T00:27:12,542 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 171e7b0d616541869910243590a09a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732148831490 2024-11-21T00:27:12,562 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#149 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:12,563 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/b77aa6e2b50f49248b8a2bec7e92e87a is 50, key is test_row_0/B:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:12,565 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:12,576 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112130fa0769dbc34021abac3bf4bb41422f_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:12,579 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112130fa0769dbc34021abac3bf4bb41422f_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:12,579 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112130fa0769dbc34021abac3bf4bb41422f_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:12,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742009_1185 (size=12983) 2024-11-21T00:27:12,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742010_1186 (size=4469) 2024-11-21T00:27:12,594 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#150 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:12,595 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/809bd4bdf4dd436d903ceb269f83827d is 175, key is test_row_0/A:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:12,604 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/b77aa6e2b50f49248b8a2bec7e92e87a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/b77aa6e2b50f49248b8a2bec7e92e87a 2024-11-21T00:27:12,613 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into b77aa6e2b50f49248b8a2bec7e92e87a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:12,613 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:12,613 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=11, startTime=1732148832533; duration=0sec 2024-11-21T00:27:12,613 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:12,613 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:12,613 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:27:12,615 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61717 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:27:12,615 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:12,617 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,617 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd64bcbda5874419a49f3a3e071f473d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0cf38d6f0a4342e79106ca23ec443e30, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d68a44f2acfc4ee992cea2f5ab014549, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0c0159ff993f4deb8e42d365cb99afbb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0f1522f6cb0b4e9684421eb94abc2bfe] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=60.3 K 2024-11-21T00:27:12,618 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting cd64bcbda5874419a49f3a3e071f473d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732148828348 2024-11-21T00:27:12,618 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cf38d6f0a4342e79106ca23ec443e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732148828746 2024-11-21T00:27:12,619 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d68a44f2acfc4ee992cea2f5ab014549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732148829529 2024-11-21T00:27:12,620 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c0159ff993f4deb8e42d365cb99afbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732148830769 2024-11-21T00:27:12,620 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f1522f6cb0b4e9684421eb94abc2bfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732148831490 2024-11-21T00:27:12,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742011_1187 (size=31937) 2024-11-21T00:27:12,650 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#151 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:12,652 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/6f14d3e60e2b48df85b36009533fa329 is 50, key is test_row_0/C:col10/1732148831500/Put/seqid=0 2024-11-21T00:27:12,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T00:27:12,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:12,658 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T00:27:12,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742012_1188 (size=12983) 2024-11-21T00:27:12,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121da1d20f231d342839484ab80c41b664b_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148831840/Put/seqid=0 2024-11-21T00:27:12,730 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/6f14d3e60e2b48df85b36009533fa329 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f14d3e60e2b48df85b36009533fa329 2024-11-21T00:27:12,740 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into 6f14d3e60e2b48df85b36009533fa329(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:12,740 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:12,740 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=11, startTime=1732148832533; duration=0sec 2024-11-21T00:27:12,740 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:12,740 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:12,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742013_1189 (size=12454) 2024-11-21T00:27:12,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:12,768 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121da1d20f231d342839484ab80c41b664b_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121da1d20f231d342839484ab80c41b664b_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/53e73d94257643a5a2c191cfbdb9d5bd, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/53e73d94257643a5a2c191cfbdb9d5bd is 175, key is test_row_0/A:col10/1732148831840/Put/seqid=0 2024-11-21T00:27:12,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742014_1190 (size=31255) 2024-11-21T00:27:12,789 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/53e73d94257643a5a2c191cfbdb9d5bd 2024-11-21T00:27:12,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/ca523e896e7f48b5a51e4a1d8843c9ab is 50, key is test_row_0/B:col10/1732148831840/Put/seqid=0 2024-11-21T00:27:12,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742015_1191 (size=12301) 2024-11-21T00:27:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:12,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:27:13,049 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/809bd4bdf4dd436d903ceb269f83827d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/809bd4bdf4dd436d903ceb269f83827d 2024-11-21T00:27:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148893041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148893043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148893044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148893046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,057 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into 809bd4bdf4dd436d903ceb269f83827d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:13,057 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:13,057 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=11, startTime=1732148832532; duration=0sec 2024-11-21T00:27:13,057 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:13,057 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:13,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148893052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148893162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148893163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148893163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148893164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148893164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,229 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/ca523e896e7f48b5a51e4a1d8843c9ab 2024-11-21T00:27:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/3ccef38b707341a087822c5173ada94d is 50, key is test_row_0/C:col10/1732148831840/Put/seqid=0 2024-11-21T00:27:13,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742016_1192 (size=12301) 2024-11-21T00:27:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148893372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148893372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148893372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148893372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148893373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,673 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/3ccef38b707341a087822c5173ada94d 2024-11-21T00:27:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/53e73d94257643a5a2c191cfbdb9d5bd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd 2024-11-21T00:27:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148893683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148893683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148893683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148893683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148893683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:13,691 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd, entries=150, sequenceid=329, filesize=30.5 K 2024-11-21T00:27:13,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/ca523e896e7f48b5a51e4a1d8843c9ab as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/ca523e896e7f48b5a51e4a1d8843c9ab 2024-11-21T00:27:13,701 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/ca523e896e7f48b5a51e4a1d8843c9ab, entries=150, sequenceid=329, filesize=12.0 K 2024-11-21T00:27:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/3ccef38b707341a087822c5173ada94d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/3ccef38b707341a087822c5173ada94d 2024-11-21T00:27:13,711 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/3ccef38b707341a087822c5173ada94d, entries=150, sequenceid=329, filesize=12.0 K 2024-11-21T00:27:13,713 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d3834c10900814f7a41633aca86edf7e in 1055ms, sequenceid=329, compaction requested=false 2024-11-21T00:27:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T00:27:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T00:27:13,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T00:27:13,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8560 sec 2024-11-21T00:27:13,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8650 sec 2024-11-21T00:27:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T00:27:14,024 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:14,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T00:27:14,026 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:27:14,027 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:14,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:27:14,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T00:27:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:14,184 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:27:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:14,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:14,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a6de63f4ba204d61b69e4af8e8b3c4ea_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148833050/Put/seqid=0 2024-11-21T00:27:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148894208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148894209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148894211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148894207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148894220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742017_1193 (size=12454) 2024-11-21T00:27:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:14,274 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a6de63f4ba204d61b69e4af8e8b3c4ea_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6de63f4ba204d61b69e4af8e8b3c4ea_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:14,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/87cef3e7a6894986a7e040295019e7c8, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/87cef3e7a6894986a7e040295019e7c8 is 175, key is test_row_0/A:col10/1732148833050/Put/seqid=0 2024-11-21T00:27:14,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742018_1194 (size=31255) 2024-11-21T00:27:14,287 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/87cef3e7a6894986a7e040295019e7c8 2024-11-21T00:27:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148894322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148894322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148894332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148894332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:27:14,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148894342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/8329e236f62a4bc2ac69d17799a0a558 is 50, key is test_row_0/B:col10/1732148833050/Put/seqid=0 2024-11-21T00:27:14,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742019_1195 (size=12301) 2024-11-21T00:27:14,422 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/8329e236f62a4bc2ac69d17799a0a558 2024-11-21T00:27:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d87c9c890eb44a078b3d3e5a6e7240c0 is 50, key is test_row_0/C:col10/1732148833050/Put/seqid=0 2024-11-21T00:27:14,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742020_1196 (size=12301) 2024-11-21T00:27:14,463 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d87c9c890eb44a078b3d3e5a6e7240c0 2024-11-21T00:27:14,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/87cef3e7a6894986a7e040295019e7c8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8 2024-11-21T00:27:14,492 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8, entries=150, sequenceid=353, filesize=30.5 K 2024-11-21T00:27:14,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/8329e236f62a4bc2ac69d17799a0a558 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/8329e236f62a4bc2ac69d17799a0a558 2024-11-21T00:27:14,502 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/8329e236f62a4bc2ac69d17799a0a558, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T00:27:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d87c9c890eb44a078b3d3e5a6e7240c0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d87c9c890eb44a078b3d3e5a6e7240c0 2024-11-21T00:27:14,509 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d87c9c890eb44a078b3d3e5a6e7240c0, entries=150, sequenceid=353, filesize=12.0 K 2024-11-21T00:27:14,514 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d3834c10900814f7a41633aca86edf7e in 330ms, sequenceid=353, compaction requested=true 2024-11-21T00:27:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T00:27:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T00:27:14,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T00:27:14,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 489 msec 2024-11-21T00:27:14,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 496 msec 2024-11-21T00:27:14,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:14,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T00:27:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:14,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112191048d1a058442738c943e9c8804ce41_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:14,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742021_1197 (size=14994) 2024-11-21T00:27:14,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148894636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148894638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148894638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T00:27:14,653 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:14,653 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T00:27:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:14,655 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:14,656 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:14,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148894645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148894646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148894753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148894753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148894753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:14,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148894763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148894764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:14,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:14,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148894962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:14,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148894968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148894968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148894972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148894982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:14,989 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112191048d1a058442738c943e9c8804ce41_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112191048d1a058442738c943e9c8804ce41_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:14,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9235363291642c99bb88c215e51449b, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:14,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9235363291642c99bb88c215e51449b is 175, key is test_row_0/A:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:15,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742022_1198 (size=39949) 2024-11-21T00:27:15,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9235363291642c99bb88c215e51449b 2024-11-21T00:27:15,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/e0761f92fa114590bc8833233e847dfb is 50, key is test_row_0/B:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:15,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742023_1199 (size=12301) 2024-11-21T00:27:15,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:15,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:15,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:15,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148895272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148895282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148895282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148895282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148895292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/e0761f92fa114590bc8833233e847dfb 2024-11-21T00:27:15,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/e8a59d13581c49c0acaa11abf2b53360 is 50, key is test_row_0/C:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:15,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742024_1200 (size=12301) 2024-11-21T00:27:15,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/e8a59d13581c49c0acaa11abf2b53360 2024-11-21T00:27:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c9235363291642c99bb88c215e51449b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b 2024-11-21T00:27:15,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b, entries=200, sequenceid=369, filesize=39.0 K 2024-11-21T00:27:15,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/e0761f92fa114590bc8833233e847dfb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/e0761f92fa114590bc8833233e847dfb 2024-11-21T00:27:15,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/e0761f92fa114590bc8833233e847dfb, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T00:27:15,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/e8a59d13581c49c0acaa11abf2b53360 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/e8a59d13581c49c0acaa11abf2b53360 2024-11-21T00:27:15,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:15,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/e8a59d13581c49c0acaa11abf2b53360, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T00:27:15,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d3834c10900814f7a41633aca86edf7e in 1061ms, sequenceid=369, compaction requested=true 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:15,596 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:27:15,596 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:15,603 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:15,603 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:15,603 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,603 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/809bd4bdf4dd436d903ceb269f83827d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=131.2 K 2024-11-21T00:27:15,603 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,603 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/809bd4bdf4dd436d903ceb269f83827d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b] 2024-11-21T00:27:15,604 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:15,604 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:15,604 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,604 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/b77aa6e2b50f49248b8a2bec7e92e87a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/ca523e896e7f48b5a51e4a1d8843c9ab, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/8329e236f62a4bc2ac69d17799a0a558, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/e0761f92fa114590bc8833233e847dfb] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=48.7 K 2024-11-21T00:27:15,604 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 809bd4bdf4dd436d903ceb269f83827d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732148831490 2024-11-21T00:27:15,605 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b77aa6e2b50f49248b8a2bec7e92e87a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732148831490 2024-11-21T00:27:15,605 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e73d94257643a5a2c191cfbdb9d5bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732148831840 2024-11-21T00:27:15,606 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ca523e896e7f48b5a51e4a1d8843c9ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732148831840 2024-11-21T00:27:15,607 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87cef3e7a6894986a7e040295019e7c8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732148833033 2024-11-21T00:27:15,607 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8329e236f62a4bc2ac69d17799a0a558, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732148833033 2024-11-21T00:27:15,607 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9235363291642c99bb88c215e51449b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148834208 2024-11-21T00:27:15,607 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e0761f92fa114590bc8833233e847dfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148834208 2024-11-21T00:27:15,617 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:15,632 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112198c1f5d2b4214df09318cb4c1a27d598_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:15,634 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:15,635 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/7c02a41980704f13b9a7eb3b1e1b3b93 is 50, key is test_row_0/B:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:15,635 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112198c1f5d2b4214df09318cb4c1a27d598_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:15,635 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112198c1f5d2b4214df09318cb4c1a27d598_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742025_1201 (size=13119) 2024-11-21T00:27:15,674 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/7c02a41980704f13b9a7eb3b1e1b3b93 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7c02a41980704f13b9a7eb3b1e1b3b93 2024-11-21T00:27:15,683 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into 7c02a41980704f13b9a7eb3b1e1b3b93(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:15,683 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:15,683 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=12, startTime=1732148835596; duration=0sec 2024-11-21T00:27:15,683 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:15,683 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:15,683 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742026_1202 (size=4469) 2024-11-21T00:27:15,687 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#161 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:15,688 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0c953c35215448f38cd1d0847981a912 is 175, key is test_row_0/A:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:15,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:15,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:15,689 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,689 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f14d3e60e2b48df85b36009533fa329, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/3ccef38b707341a087822c5173ada94d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d87c9c890eb44a078b3d3e5a6e7240c0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/e8a59d13581c49c0acaa11abf2b53360] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=48.7 K 2024-11-21T00:27:15,690 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f14d3e60e2b48df85b36009533fa329, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732148831490 2024-11-21T00:27:15,690 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ccef38b707341a087822c5173ada94d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732148831840 2024-11-21T00:27:15,691 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d87c9c890eb44a078b3d3e5a6e7240c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732148833033 2024-11-21T00:27:15,692 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a59d13581c49c0acaa11abf2b53360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148834208 2024-11-21T00:27:15,713 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:15,714 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/5001562f15134c3089d8de19c593f944 is 50, key is test_row_0/C:col10/1732148834210/Put/seqid=0 2024-11-21T00:27:15,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742027_1203 (size=32073) 2024-11-21T00:27:15,725 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0c953c35215448f38cd1d0847981a912 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0c953c35215448f38cd1d0847981a912 2024-11-21T00:27:15,732 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into 0c953c35215448f38cd1d0847981a912(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:15,732 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:15,732 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=12, startTime=1732148835596; duration=0sec 2024-11-21T00:27:15,732 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:15,732 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:15,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T00:27:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:15,739 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T00:27:15,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742028_1204 (size=13119) 2024-11-21T00:27:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd7ec88a74c34667a207073ea9c6bf97_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148834632/Put/seqid=0 2024-11-21T00:27:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:15,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. as already flushing 2024-11-21T00:27:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742029_1205 (size=12454) 2024-11-21T00:27:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148895829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148895831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148895832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148895833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148895832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,845 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd7ec88a74c34667a207073ea9c6bf97_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd7ec88a74c34667a207073ea9c6bf97_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:15,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c5fe89e8b8fa4ff98287acf651aa9fca, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:15,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c5fe89e8b8fa4ff98287acf651aa9fca is 175, key is test_row_0/A:col10/1732148834632/Put/seqid=0 2024-11-21T00:27:15,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742030_1206 (size=31255) 2024-11-21T00:27:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148895942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148895942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148895952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148895952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148895952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,006 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:27:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148896152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148896152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148896163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148896162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148896164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,182 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/5001562f15134c3089d8de19c593f944 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/5001562f15134c3089d8de19c593f944 2024-11-21T00:27:16,189 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into 5001562f15134c3089d8de19c593f944(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:16,189 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:16,189 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=12, startTime=1732148835596; duration=0sec 2024-11-21T00:27:16,189 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:16,189 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:16,286 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c5fe89e8b8fa4ff98287acf651aa9fca 2024-11-21T00:27:16,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/7da57ca43d5a42969f81bded91a41685 is 50, key is test_row_0/B:col10/1732148834632/Put/seqid=0 2024-11-21T00:27:16,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742031_1207 (size=12301) 2024-11-21T00:27:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148896473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148896473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148896473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148896473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148896473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,780 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/7da57ca43d5a42969f81bded91a41685 2024-11-21T00:27:16,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b6a52eafc32145a784c1c794ef6fbcdc is 50, key is test_row_0/C:col10/1732148834632/Put/seqid=0 2024-11-21T00:27:16,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742032_1208 (size=12301) 2024-11-21T00:27:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:16,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148896992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148896992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148896992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:16,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148896993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:17,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148896993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:17,201 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b6a52eafc32145a784c1c794ef6fbcdc 2024-11-21T00:27:17,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/c5fe89e8b8fa4ff98287acf651aa9fca as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca 2024-11-21T00:27:17,214 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca, entries=150, sequenceid=391, filesize=30.5 K 2024-11-21T00:27:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/7da57ca43d5a42969f81bded91a41685 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7da57ca43d5a42969f81bded91a41685 2024-11-21T00:27:17,236 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7da57ca43d5a42969f81bded91a41685, entries=150, sequenceid=391, filesize=12.0 K 2024-11-21T00:27:17,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/b6a52eafc32145a784c1c794ef6fbcdc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b6a52eafc32145a784c1c794ef6fbcdc 2024-11-21T00:27:17,254 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b6a52eafc32145a784c1c794ef6fbcdc, entries=150, sequenceid=391, filesize=12.0 K 2024-11-21T00:27:17,255 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d3834c10900814f7a41633aca86edf7e in 1516ms, sequenceid=391, compaction requested=false 2024-11-21T00:27:17,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:17,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:17,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T00:27:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T00:27:17,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T00:27:17,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6020 sec 2024-11-21T00:27:17,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6100 sec 2024-11-21T00:27:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:18,008 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:18,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:27:18,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121752bc558853048eaa7678694f9085c9e_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148898030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148898034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148898032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148898040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148898041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742033_1209 (size=14994) 2024-11-21T00:27:18,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:18,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121752bc558853048eaa7678694f9085c9e_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121752bc558853048eaa7678694f9085c9e_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:18,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0e3435210a704800a5e28af7c79c1b7f, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:18,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0e3435210a704800a5e28af7c79c1b7f is 175, key is test_row_0/A:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742034_1210 (size=39949) 2024-11-21T00:27:18,114 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0e3435210a704800a5e28af7c79c1b7f 2024-11-21T00:27:18,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/298798bdfacd4343b184848d4704d838 is 50, key is test_row_0/B:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1732148898152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148898153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45608 deadline: 1732148898153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148898153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1732148898153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742035_1211 (size=12301) 2024-11-21T00:27:18,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/298798bdfacd4343b184848d4704d838 2024-11-21T00:27:18,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/6f6922cfa4284189b1534b65e54f713a is 50, key is test_row_0/C:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742036_1212 (size=12301) 2024-11-21T00:27:18,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/6f6922cfa4284189b1534b65e54f713a 2024-11-21T00:27:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/0e3435210a704800a5e28af7c79c1b7f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f 2024-11-21T00:27:18,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f, entries=200, sequenceid=412, filesize=39.0 K 2024-11-21T00:27:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/298798bdfacd4343b184848d4704d838 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/298798bdfacd4343b184848d4704d838 2024-11-21T00:27:18,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/298798bdfacd4343b184848d4704d838, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T00:27:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/6f6922cfa4284189b1534b65e54f713a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f6922cfa4284189b1534b65e54f713a 2024-11-21T00:27:18,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f6922cfa4284189b1534b65e54f713a, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T00:27:18,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d3834c10900814f7a41633aca86edf7e in 227ms, sequenceid=412, compaction requested=true 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:18,235 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:18,235 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:18,238 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:18,238 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:18,239 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/B is initiating minor compaction (all files) 2024-11-21T00:27:18,239 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/B in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:18,239 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7c02a41980704f13b9a7eb3b1e1b3b93, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7da57ca43d5a42969f81bded91a41685, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/298798bdfacd4343b184848d4704d838] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=36.8 K 2024-11-21T00:27:18,239 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/A is initiating minor compaction (all files) 2024-11-21T00:27:18,239 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/A in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:18,239 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0c953c35215448f38cd1d0847981a912, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=100.9 K 2024-11-21T00:27:18,239 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:18,239 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0c953c35215448f38cd1d0847981a912, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f] 2024-11-21T00:27:18,240 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c02a41980704f13b9a7eb3b1e1b3b93, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148834208 2024-11-21T00:27:18,240 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da57ca43d5a42969f81bded91a41685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732148834632 2024-11-21T00:27:18,241 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c953c35215448f38cd1d0847981a912, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148834208 2024-11-21T00:27:18,242 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 298798bdfacd4343b184848d4704d838, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732148835830 2024-11-21T00:27:18,242 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5fe89e8b8fa4ff98287acf651aa9fca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732148834632 2024-11-21T00:27:18,243 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3435210a704800a5e28af7c79c1b7f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732148835830 2024-11-21T00:27:18,254 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#B#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:18,255 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/9c6420a75e0941308a2335207c023aa9 is 50, key is test_row_0/B:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,258 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:18,276 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210b1c4e62c34d4154ad5f9b78179a7f6e_d3834c10900814f7a41633aca86edf7e store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:18,279 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210b1c4e62c34d4154ad5f9b78179a7f6e_d3834c10900814f7a41633aca86edf7e, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:18,279 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210b1c4e62c34d4154ad5f9b78179a7f6e_d3834c10900814f7a41633aca86edf7e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:18,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742037_1213 (size=13221) 2024-11-21T00:27:18,318 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/9c6420a75e0941308a2335207c023aa9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/9c6420a75e0941308a2335207c023aa9 2024-11-21T00:27:18,326 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3834c10900814f7a41633aca86edf7e/B of d3834c10900814f7a41633aca86edf7e into 9c6420a75e0941308a2335207c023aa9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:18,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:18,326 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/B, priority=13, startTime=1732148838235; duration=0sec 2024-11-21T00:27:18,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:18,327 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:18,327 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:18,328 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:18,328 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): d3834c10900814f7a41633aca86edf7e/C is initiating minor compaction (all files) 2024-11-21T00:27:18,328 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3834c10900814f7a41633aca86edf7e/C in TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:18,328 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/5001562f15134c3089d8de19c593f944, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b6a52eafc32145a784c1c794ef6fbcdc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f6922cfa4284189b1534b65e54f713a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp, totalSize=36.8 K 2024-11-21T00:27:18,329 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5001562f15134c3089d8de19c593f944, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148834208 2024-11-21T00:27:18,329 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a52eafc32145a784c1c794ef6fbcdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732148834632 2024-11-21T00:27:18,330 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f6922cfa4284189b1534b65e54f713a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732148835830 2024-11-21T00:27:18,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742038_1214 (size=4469) 2024-11-21T00:27:18,344 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#C#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:18,344 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/c54f6455030c4dbba5a69f31bf43d3bd is 50, key is test_row_0/C:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,354 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3834c10900814f7a41633aca86edf7e#A#compaction#171 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:18,355 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fe01c6578f3a421ea4e1dd94338d58a4 is 175, key is test_row_0/A:col10/1732148835831/Put/seqid=0 2024-11-21T00:27:18,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742039_1215 (size=13221) 2024-11-21T00:27:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:18,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:18,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:27:18,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742040_1216 (size=32175) 2024-11-21T00:27:18,395 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/c54f6455030c4dbba5a69f31bf43d3bd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c54f6455030c4dbba5a69f31bf43d3bd 2024-11-21T00:27:18,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a53d926ecb3540e08ece2e473d7ee97b_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148838366/Put/seqid=0 2024-11-21T00:27:18,401 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fe01c6578f3a421ea4e1dd94338d58a4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fe01c6578f3a421ea4e1dd94338d58a4 2024-11-21T00:27:18,407 INFO [Thread-559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-559. 2024-11-21T00:27:18,407 DEBUG [Thread-559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,407 DEBUG [Thread-559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,407 DEBUG [Thread-559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,448 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3834c10900814f7a41633aca86edf7e/C of d3834c10900814f7a41633aca86edf7e into c54f6455030c4dbba5a69f31bf43d3bd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:18,448 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:18,448 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/C, priority=13, startTime=1732148838235; duration=0sec 2024-11-21T00:27:18,449 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3834c10900814f7a41633aca86edf7e/A of d3834c10900814f7a41633aca86edf7e into fe01c6578f3a421ea4e1dd94338d58a4(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:18,449 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:18,449 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., storeName=d3834c10900814f7a41633aca86edf7e/A, priority=13, startTime=1732148838235; duration=0sec 2024-11-21T00:27:18,449 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:18,449 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:18,449 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:18,449 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:18,449 INFO [Thread-552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-552. 2024-11-21T00:27:18,449 INFO [Thread-550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-550. 2024-11-21T00:27:18,449 DEBUG [Thread-552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,450 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,450 DEBUG [Thread-550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,450 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,450 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,450 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,450 INFO [Thread-554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-554. 2024-11-21T00:27:18,450 DEBUG [Thread-554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,451 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,451 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742041_1217 (size=14994) 2024-11-21T00:27:18,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148898404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148898405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,459 INFO [Thread-561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-561. 2024-11-21T00:27:18,459 DEBUG [Thread-561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,459 DEBUG [Thread-561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,459 DEBUG [Thread-561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,466 INFO [Thread-565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-565. 2024-11-21T00:27:18,466 DEBUG [Thread-565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,466 DEBUG [Thread-565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,466 DEBUG [Thread-565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,467 INFO [Thread-563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-563. 2024-11-21T00:27:18,467 DEBUG [Thread-563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:18,467 DEBUG [Thread-563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,468 DEBUG [Thread-563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:18,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:18,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148898562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148898572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148898772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148898782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T00:27:18,825 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:18,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:18,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a53d926ecb3540e08ece2e473d7ee97b_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a53d926ecb3540e08ece2e473d7ee97b_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:18,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/67378df27ee845deb024e0fecbf4a7a7, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:18,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/67378df27ee845deb024e0fecbf4a7a7 is 175, key is test_row_0/A:col10/1732148838366/Put/seqid=0 2024-11-21T00:27:18,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742042_1218 (size=39949) 2024-11-21T00:27:19,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148899083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:19,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148899093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:19,292 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/67378df27ee845deb024e0fecbf4a7a7 2024-11-21T00:27:19,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/fdd8274e72d646c387932bd743ab94f7 is 50, key is test_row_0/B:col10/1732148838366/Put/seqid=0 2024-11-21T00:27:19,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742043_1219 (size=12301) 2024-11-21T00:27:19,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/fdd8274e72d646c387932bd743ab94f7 2024-11-21T00:27:19,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d97853d02cef42609d800bb3397e487d is 50, key is test_row_0/C:col10/1732148838366/Put/seqid=0 2024-11-21T00:27:19,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742044_1220 (size=12301) 2024-11-21T00:27:19,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1732148899592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45628 deadline: 1732148899603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:19,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d97853d02cef42609d800bb3397e487d 2024-11-21T00:27:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/67378df27ee845deb024e0fecbf4a7a7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67378df27ee845deb024e0fecbf4a7a7 2024-11-21T00:27:19,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67378df27ee845deb024e0fecbf4a7a7, entries=200, sequenceid=431, filesize=39.0 K 2024-11-21T00:27:19,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/fdd8274e72d646c387932bd743ab94f7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/fdd8274e72d646c387932bd743ab94f7 2024-11-21T00:27:19,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/fdd8274e72d646c387932bd743ab94f7, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:27:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/d97853d02cef42609d800bb3397e487d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d97853d02cef42609d800bb3397e487d 2024-11-21T00:27:19,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d97853d02cef42609d800bb3397e487d, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T00:27:19,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d3834c10900814f7a41633aca86edf7e in 1427ms, sequenceid=431, compaction requested=false 2024-11-21T00:27:19,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:20,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:27:20,604 INFO [Thread-548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-548. 2024-11-21T00:27:20,604 DEBUG [Thread-548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:20,604 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:20,604 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:20,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:20,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112178f3fc07a8e9447080d600f07c087b07_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_0/A:col10/1732148838403/Put/seqid=0 2024-11-21T00:27:20,615 INFO [Thread-556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-556. 2024-11-21T00:27:20,615 DEBUG [Thread-556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:20,615 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:20,615 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:20,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2204 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2189 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3147 rows 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-11-21T00:27:20,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3177 rows 2024-11-21T00:27:20,615 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:27:20,615 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:27:20,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:20,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:20,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:20,616 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:27:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:20,620 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148840620"}]},"ts":"1732148840620"} 2024-11-21T00:27:20,622 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:27:20,623 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:27:20,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:27:20,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, UNASSIGN}] 2024-11-21T00:27:20,627 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, UNASSIGN 2024-11-21T00:27:20,628 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:20,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, UNASSIGN because future has completed 2024-11-21T00:27:20,630 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:27:20,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:27:20,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742045_1221 (size=12454) 2024-11-21T00:27:20,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:20,645 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112178f3fc07a8e9447080d600f07c087b07_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112178f3fc07a8e9447080d600f07c087b07_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:20,647 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fc4e39af5d784b38b04b6b9cb5317401, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:20,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fc4e39af5d784b38b04b6b9cb5317401 is 175, key is test_row_0/A:col10/1732148838403/Put/seqid=0 2024-11-21T00:27:20,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742046_1222 (size=31255) 2024-11-21T00:27:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:20,783 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:20,783 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:27:20,783 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing d3834c10900814f7a41633aca86edf7e, disabling compactions & flushes 2024-11-21T00:27:20,783 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:21,076 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fc4e39af5d784b38b04b6b9cb5317401 2024-11-21T00:27:21,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/383ee51bcf8b4e9bbcb5669b3efc2451 is 50, key is test_row_0/B:col10/1732148838403/Put/seqid=0 2024-11-21T00:27:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742047_1223 (size=12301) 2024-11-21T00:27:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:21,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/383ee51bcf8b4e9bbcb5669b3efc2451 2024-11-21T00:27:21,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/4d7dcb2edc834cceb495cd756a2e8897 is 50, key is test_row_0/C:col10/1732148838403/Put/seqid=0 2024-11-21T00:27:21,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742048_1224 (size=12301) 2024-11-21T00:27:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:21,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/4d7dcb2edc834cceb495cd756a2e8897 2024-11-21T00:27:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/fc4e39af5d784b38b04b6b9cb5317401 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fc4e39af5d784b38b04b6b9cb5317401 2024-11-21T00:27:21,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fc4e39af5d784b38b04b6b9cb5317401, entries=150, sequenceid=452, filesize=30.5 K 2024-11-21T00:27:21,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/383ee51bcf8b4e9bbcb5669b3efc2451 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/383ee51bcf8b4e9bbcb5669b3efc2451 2024-11-21T00:27:21,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/383ee51bcf8b4e9bbcb5669b3efc2451, entries=150, sequenceid=452, filesize=12.0 K 2024-11-21T00:27:21,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/4d7dcb2edc834cceb495cd756a2e8897 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/4d7dcb2edc834cceb495cd756a2e8897 2024-11-21T00:27:21,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/4d7dcb2edc834cceb495cd756a2e8897, entries=150, sequenceid=452, filesize=12.0 K 2024-11-21T00:27:21,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=6.71 KB/6870 for d3834c10900814f7a41633aca86edf7e in 1395ms, sequenceid=452, compaction requested=true 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3834c10900814f7a41633aca86edf7e: 2024-11-21T00:27:22,000 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:22,000 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:22,000 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. because compaction request was cancelled 2024-11-21T00:27:22,000 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:A 2024-11-21T00:27:22,000 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. after waiting 0 ms 2024-11-21T00:27:22,000 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:A, priority=-2147483648, current under compaction store size is 0 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:22,000 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing d3834c10900814f7a41633aca86edf7e 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:B, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:22,000 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. because compaction request was cancelled 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:22,000 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:B 2024-11-21T00:27:22,000 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. because compaction request was cancelled 2024-11-21T00:27:22,000 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3834c10900814f7a41633aca86edf7e:C 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3834c10900814f7a41633aca86edf7e:C, priority=-2147483648, current under compaction store size is 0 2024-11-21T00:27:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:22,013 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b59d17a3f1e541aa82755a32a06e8c72_d3834c10900814f7a41633aca86edf7e is 50, key is test_row_2/A:col10/1732148840612/Put/seqid=0 2024-11-21T00:27:22,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742049_1225 (size=7374) 2024-11-21T00:27:22,420 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:22,425 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b59d17a3f1e541aa82755a32a06e8c72_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b59d17a3f1e541aa82755a32a06e8c72_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:22,429 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2, store: [table=TestAcidGuarantees family=A region=d3834c10900814f7a41633aca86edf7e] 2024-11-21T00:27:22,430 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2 is 175, key is test_row_2/A:col10/1732148840612/Put/seqid=0 2024-11-21T00:27:22,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742050_1226 (size=13865) 2024-11-21T00:27:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:22,837 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2 2024-11-21T00:27:22,843 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/a62f4bc00a4f4e48bda7727008317a23 is 50, key is test_row_2/B:col10/1732148840612/Put/seqid=0 2024-11-21T00:27:22,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742051_1227 (size=7415) 2024-11-21T00:27:23,247 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/a62f4bc00a4f4e48bda7727008317a23 2024-11-21T00:27:23,259 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/53da8b8600a24650a0bda03d70192a1b is 50, key is test_row_2/C:col10/1732148840612/Put/seqid=0 2024-11-21T00:27:23,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742052_1228 (size=7415) 2024-11-21T00:27:23,266 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/53da8b8600a24650a0bda03d70192a1b 2024-11-21T00:27:23,273 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2 2024-11-21T00:27:23,279 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2, entries=50, sequenceid=456, filesize=13.5 K 2024-11-21T00:27:23,281 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/B/a62f4bc00a4f4e48bda7727008317a23 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/a62f4bc00a4f4e48bda7727008317a23 2024-11-21T00:27:23,286 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/a62f4bc00a4f4e48bda7727008317a23, entries=50, sequenceid=456, filesize=7.2 K 2024-11-21T00:27:23,288 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/.tmp/C/53da8b8600a24650a0bda03d70192a1b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/53da8b8600a24650a0bda03d70192a1b 2024-11-21T00:27:23,295 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/53da8b8600a24650a0bda03d70192a1b, entries=50, sequenceid=456, filesize=7.2 K 2024-11-21T00:27:23,296 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d3834c10900814f7a41633aca86edf7e in 1296ms, sequenceid=456, compaction requested=true 2024-11-21T00:27:23,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/cfa245005c2b44fc89a119c485ebdbf1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/a6af7586e0d14d928d3fa52f4f041099, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67cbfc0624d545e19616c30f106b6581, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b2ebe8b922194be8a2a848cbf919e52f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/809bd4bdf4dd436d903ceb269f83827d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0c953c35215448f38cd1d0847981a912, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f] to archive 2024-11-21T00:27:23,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:27:23,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/71ea47706f88417f897ac03e7050dd52 2024-11-21T00:27:23,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e6e3254485ff4aa1bbd27cc7a79da1d5 2024-11-21T00:27:23,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/cfa245005c2b44fc89a119c485ebdbf1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/cfa245005c2b44fc89a119c485ebdbf1 2024-11-21T00:27:23,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/373b7fc5cb244ce69503cfd5114aac2f 2024-11-21T00:27:23,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/3ba935e08c5442fea2c97410375d50be 2024-11-21T00:27:23,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/7d64140aefec46febf13b00b7234b8ec 2024-11-21T00:27:23,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fa4afba19b1342a4b224deb1182f29b6 2024-11-21T00:27:23,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/a6af7586e0d14d928d3fa52f4f041099 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/a6af7586e0d14d928d3fa52f4f041099 2024-11-21T00:27:23,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/594d470bca9449be9d0e2dc6eff8847e 2024-11-21T00:27:23,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c45bebeccc54412ba39d7f33d92765de 2024-11-21T00:27:23,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/392313474d2e4777924512d6b0f2baec 2024-11-21T00:27:23,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67cbfc0624d545e19616c30f106b6581 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67cbfc0624d545e19616c30f106b6581 2024-11-21T00:27:23,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b27967ec3006495ebefa6e4be2b51f19 2024-11-21T00:27:23,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/e1c42c7dc34843db90c9b7dc5b405b18 2024-11-21T00:27:23,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9b558da147449408c40359140414556 2024-11-21T00:27:23,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b2ebe8b922194be8a2a848cbf919e52f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/b2ebe8b922194be8a2a848cbf919e52f 2024-11-21T00:27:23,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/5862a0f2a0de44efae4172d867af9bd1 2024-11-21T00:27:23,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0970417e741748c699f2557ca6859565 2024-11-21T00:27:23,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/83fb521292bb4300971fa263a68b3e37 2024-11-21T00:27:23,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/232f2c0f80e24a45a197b3ce8182ff9a 2024-11-21T00:27:23,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/809bd4bdf4dd436d903ceb269f83827d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/809bd4bdf4dd436d903ceb269f83827d 2024-11-21T00:27:23,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/53e73d94257643a5a2c191cfbdb9d5bd 2024-11-21T00:27:23,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/87cef3e7a6894986a7e040295019e7c8 2024-11-21T00:27:23,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c9235363291642c99bb88c215e51449b 2024-11-21T00:27:23,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0c953c35215448f38cd1d0847981a912 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0c953c35215448f38cd1d0847981a912 2024-11-21T00:27:23,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/c5fe89e8b8fa4ff98287acf651aa9fca 2024-11-21T00:27:23,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/0e3435210a704800a5e28af7c79c1b7f 2024-11-21T00:27:23,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/362f46cef6164b958a005dd0dc0b3d25, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/75a91be9321f4f72b6c516af3b4bcf9d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f4c102eeaa2045c3bc7743d8981013b9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/c655eaddf7b44d0e851ccfd472253f4a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/710983c05dea42d39795ca84b9789041, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/83993c04962a48c099e50ec887d8809a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f1c18cf63a5945f78e9cba8ea4e2edf6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/11b880494d4c450ba3df28b7026175b8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6a172523da514e51b958f8e3afe491bf, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3792c8f0211444e0a8a9c9002fa32ff8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3312135ac6d64da5b0c3bd4f12620bfb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/d85801383e5044e3be8589816919f84b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/90eb60a68a8e494aaf50bb35bdc38eec, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/88dc45c3962b475f807c0435ed372a87, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6f48cd8eb08d42ddb9bc427c82238dc7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3384fafc042a4b349db82c0d96c46178, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/0250e751773f47b1b9c1a71edf201a87, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/bda3fa5e9e0f46b4845b0e8f952882c2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/831125ce47dd444fb00cba81fa89c54c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/b77aa6e2b50f49248b8a2bec7e92e87a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/171e7b0d616541869910243590a09a1a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/ca523e896e7f48b5a51e4a1d8843c9ab, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/8329e236f62a4bc2ac69d17799a0a558, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7c02a41980704f13b9a7eb3b1e1b3b93, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/e0761f92fa114590bc8833233e847dfb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7da57ca43d5a42969f81bded91a41685, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/298798bdfacd4343b184848d4704d838] to archive 2024-11-21T00:27:23,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:27:23,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/362f46cef6164b958a005dd0dc0b3d25 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/362f46cef6164b958a005dd0dc0b3d25 2024-11-21T00:27:23,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/75a91be9321f4f72b6c516af3b4bcf9d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/75a91be9321f4f72b6c516af3b4bcf9d 2024-11-21T00:27:23,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f4c102eeaa2045c3bc7743d8981013b9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f4c102eeaa2045c3bc7743d8981013b9 2024-11-21T00:27:23,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/c655eaddf7b44d0e851ccfd472253f4a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/c655eaddf7b44d0e851ccfd472253f4a 2024-11-21T00:27:23,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/710983c05dea42d39795ca84b9789041 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/710983c05dea42d39795ca84b9789041 2024-11-21T00:27:23,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/83993c04962a48c099e50ec887d8809a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/83993c04962a48c099e50ec887d8809a 2024-11-21T00:27:23,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f1c18cf63a5945f78e9cba8ea4e2edf6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/f1c18cf63a5945f78e9cba8ea4e2edf6 2024-11-21T00:27:23,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/11b880494d4c450ba3df28b7026175b8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/11b880494d4c450ba3df28b7026175b8 2024-11-21T00:27:23,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6a172523da514e51b958f8e3afe491bf to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6a172523da514e51b958f8e3afe491bf 2024-11-21T00:27:23,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3792c8f0211444e0a8a9c9002fa32ff8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3792c8f0211444e0a8a9c9002fa32ff8 2024-11-21T00:27:23,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3312135ac6d64da5b0c3bd4f12620bfb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3312135ac6d64da5b0c3bd4f12620bfb 2024-11-21T00:27:23,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/d85801383e5044e3be8589816919f84b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/d85801383e5044e3be8589816919f84b 2024-11-21T00:27:23,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/90eb60a68a8e494aaf50bb35bdc38eec to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/90eb60a68a8e494aaf50bb35bdc38eec 2024-11-21T00:27:23,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/88dc45c3962b475f807c0435ed372a87 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/88dc45c3962b475f807c0435ed372a87 2024-11-21T00:27:23,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6f48cd8eb08d42ddb9bc427c82238dc7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/6f48cd8eb08d42ddb9bc427c82238dc7 2024-11-21T00:27:23,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3384fafc042a4b349db82c0d96c46178 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/3384fafc042a4b349db82c0d96c46178 2024-11-21T00:27:23,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/0250e751773f47b1b9c1a71edf201a87 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/0250e751773f47b1b9c1a71edf201a87 2024-11-21T00:27:23,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/bda3fa5e9e0f46b4845b0e8f952882c2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/bda3fa5e9e0f46b4845b0e8f952882c2 2024-11-21T00:27:23,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/831125ce47dd444fb00cba81fa89c54c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/831125ce47dd444fb00cba81fa89c54c 2024-11-21T00:27:23,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/b77aa6e2b50f49248b8a2bec7e92e87a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/b77aa6e2b50f49248b8a2bec7e92e87a 2024-11-21T00:27:23,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/171e7b0d616541869910243590a09a1a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/171e7b0d616541869910243590a09a1a 2024-11-21T00:27:23,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/ca523e896e7f48b5a51e4a1d8843c9ab to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/ca523e896e7f48b5a51e4a1d8843c9ab 2024-11-21T00:27:23,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/8329e236f62a4bc2ac69d17799a0a558 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/8329e236f62a4bc2ac69d17799a0a558 2024-11-21T00:27:23,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7c02a41980704f13b9a7eb3b1e1b3b93 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7c02a41980704f13b9a7eb3b1e1b3b93 2024-11-21T00:27:23,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/e0761f92fa114590bc8833233e847dfb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/e0761f92fa114590bc8833233e847dfb 2024-11-21T00:27:23,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7da57ca43d5a42969f81bded91a41685 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/7da57ca43d5a42969f81bded91a41685 2024-11-21T00:27:23,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/298798bdfacd4343b184848d4704d838 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/298798bdfacd4343b184848d4704d838 2024-11-21T00:27:23,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b1e448237c3b4b36aa9b09cbeb4c3aa5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/19d2677ac9b243f0bad4ea3bac9ec4a9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/03107a4194764aba8fcdab7b93bf8c5b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b8a0099cc3724787a9e1aa46ab5d840b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/2189378c707449ad95a4e32c764afaca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c77c7f4356af463e93625b0d9f60ddb7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/97b81ce4874e4f8397aee10179f4a93b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/fc9b931fae2e4db587335953c39183c3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/7bf258c9a74444e7af37fd4df44cfce9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/440d392cef8b4d518127ec5f797f120b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bf530600369146f7a376956846aa4464, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/ec9f69facf9d417e9c1fd48262880a2f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bb24edc6d4d741e6a051f6405933a562, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/de095100656d4a0699467b8e3948a8e8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd64bcbda5874419a49f3a3e071f473d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd9f4c1a40e54d459eff7853d27aa928, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0cf38d6f0a4342e79106ca23ec443e30, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d68a44f2acfc4ee992cea2f5ab014549, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0c0159ff993f4deb8e42d365cb99afbb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f14d3e60e2b48df85b36009533fa329, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0f1522f6cb0b4e9684421eb94abc2bfe, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/3ccef38b707341a087822c5173ada94d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d87c9c890eb44a078b3d3e5a6e7240c0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/5001562f15134c3089d8de19c593f944, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/e8a59d13581c49c0acaa11abf2b53360, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b6a52eafc32145a784c1c794ef6fbcdc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f6922cfa4284189b1534b65e54f713a] to archive 2024-11-21T00:27:23,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:27:23,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b1e448237c3b4b36aa9b09cbeb4c3aa5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b1e448237c3b4b36aa9b09cbeb4c3aa5 2024-11-21T00:27:23,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/19d2677ac9b243f0bad4ea3bac9ec4a9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/19d2677ac9b243f0bad4ea3bac9ec4a9 2024-11-21T00:27:23,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/03107a4194764aba8fcdab7b93bf8c5b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/03107a4194764aba8fcdab7b93bf8c5b 2024-11-21T00:27:23,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b8a0099cc3724787a9e1aa46ab5d840b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b8a0099cc3724787a9e1aa46ab5d840b 2024-11-21T00:27:23,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/2189378c707449ad95a4e32c764afaca to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/2189378c707449ad95a4e32c764afaca 2024-11-21T00:27:23,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c77c7f4356af463e93625b0d9f60ddb7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c77c7f4356af463e93625b0d9f60ddb7 2024-11-21T00:27:23,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/97b81ce4874e4f8397aee10179f4a93b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/97b81ce4874e4f8397aee10179f4a93b 2024-11-21T00:27:23,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/fc9b931fae2e4db587335953c39183c3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/fc9b931fae2e4db587335953c39183c3 2024-11-21T00:27:23,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/7bf258c9a74444e7af37fd4df44cfce9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/7bf258c9a74444e7af37fd4df44cfce9 2024-11-21T00:27:23,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/440d392cef8b4d518127ec5f797f120b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/440d392cef8b4d518127ec5f797f120b 2024-11-21T00:27:23,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bf530600369146f7a376956846aa4464 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bf530600369146f7a376956846aa4464 2024-11-21T00:27:23,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/ec9f69facf9d417e9c1fd48262880a2f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/ec9f69facf9d417e9c1fd48262880a2f 2024-11-21T00:27:23,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bb24edc6d4d741e6a051f6405933a562 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/bb24edc6d4d741e6a051f6405933a562 2024-11-21T00:27:23,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/de095100656d4a0699467b8e3948a8e8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/de095100656d4a0699467b8e3948a8e8 2024-11-21T00:27:23,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd64bcbda5874419a49f3a3e071f473d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd64bcbda5874419a49f3a3e071f473d 2024-11-21T00:27:23,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd9f4c1a40e54d459eff7853d27aa928 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/cd9f4c1a40e54d459eff7853d27aa928 2024-11-21T00:27:23,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0cf38d6f0a4342e79106ca23ec443e30 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0cf38d6f0a4342e79106ca23ec443e30 2024-11-21T00:27:23,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d68a44f2acfc4ee992cea2f5ab014549 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d68a44f2acfc4ee992cea2f5ab014549 2024-11-21T00:27:23,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0c0159ff993f4deb8e42d365cb99afbb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0c0159ff993f4deb8e42d365cb99afbb 2024-11-21T00:27:23,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f14d3e60e2b48df85b36009533fa329 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f14d3e60e2b48df85b36009533fa329 2024-11-21T00:27:23,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0f1522f6cb0b4e9684421eb94abc2bfe to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/0f1522f6cb0b4e9684421eb94abc2bfe 2024-11-21T00:27:23,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/3ccef38b707341a087822c5173ada94d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/3ccef38b707341a087822c5173ada94d 2024-11-21T00:27:23,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d87c9c890eb44a078b3d3e5a6e7240c0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d87c9c890eb44a078b3d3e5a6e7240c0 2024-11-21T00:27:23,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/5001562f15134c3089d8de19c593f944 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/5001562f15134c3089d8de19c593f944 2024-11-21T00:27:23,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/e8a59d13581c49c0acaa11abf2b53360 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/e8a59d13581c49c0acaa11abf2b53360 2024-11-21T00:27:23,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b6a52eafc32145a784c1c794ef6fbcdc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/b6a52eafc32145a784c1c794ef6fbcdc 2024-11-21T00:27:23,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f6922cfa4284189b1534b65e54f713a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/6f6922cfa4284189b1534b65e54f713a 2024-11-21T00:27:23,613 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/recovered.edits/459.seqid, newMaxSeqId=459, maxSeqId=4 2024-11-21T00:27:23,614 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. 2024-11-21T00:27:23,614 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for d3834c10900814f7a41633aca86edf7e: Waiting for close lock at 1732148840783Running coprocessor pre-close hooks at 1732148840783Disabling compacts and flushes for region at 1732148840783Disabling writes for close at 1732148842000 (+1217 ms)Obtaining lock to block concurrent updates at 1732148842000Preparing flush snapshotting stores in d3834c10900814f7a41633aca86edf7e at 1732148842000Finished memstore snapshotting TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732148842000Flushing stores of TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e. at 1732148842001 (+1 ms)Flushing d3834c10900814f7a41633aca86edf7e/A: creating writer at 1732148842001Flushing mob file d3834c10900814f7a41633aca86edf7e/A: appending metadata at 1732148842013 (+12 ms)Flushing mob file d3834c10900814f7a41633aca86edf7e/A: closing flushed file at 1732148842013Flushing d3834c10900814f7a41633aca86edf7e/A: appending metadata at 1732148842429 (+416 ms)Flushing d3834c10900814f7a41633aca86edf7e/A: closing flushed file at 1732148842429Flushing d3834c10900814f7a41633aca86edf7e/B: creating writer at 1732148842841 (+412 ms)Flushing d3834c10900814f7a41633aca86edf7e/B: appending metadata at 1732148842843 (+2 ms)Flushing d3834c10900814f7a41633aca86edf7e/B: closing flushed file at 1732148842843Flushing d3834c10900814f7a41633aca86edf7e/C: creating writer at 1732148843253 (+410 ms)Flushing d3834c10900814f7a41633aca86edf7e/C: appending metadata at 1732148843258 (+5 ms)Flushing d3834c10900814f7a41633aca86edf7e/C: closing flushed file at 1732148843258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29fba18a: reopening flushed file at 1732148843271 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5593b12e: reopening flushed file at 1732148843279 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@768b16c5: reopening flushed file at 1732148843286 (+7 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d3834c10900814f7a41633aca86edf7e in 1296ms, sequenceid=456, compaction requested=true at 1732148843296 (+10 ms)Writing region close event to WAL at 1732148843599 (+303 ms)Running coprocessor post-close hooks at 1732148843614 (+15 ms)Closed at 1732148843614 2024-11-21T00:27:23,616 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:23,617 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=d3834c10900814f7a41633aca86edf7e, regionState=CLOSED 2024-11-21T00:27:23,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:27:23,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T00:27:23,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure d3834c10900814f7a41633aca86edf7e, server=5ed4808ef0e6,38589,1732148778087 in 2.9890 sec 2024-11-21T00:27:23,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-21T00:27:23,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3834c10900814f7a41633aca86edf7e, UNASSIGN in 2.9950 sec 2024-11-21T00:27:23,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T00:27:23,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.0000 sec 2024-11-21T00:27:23,627 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148843627"}]},"ts":"1732148843627"} 2024-11-21T00:27:23,628 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:27:23,628 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:27:23,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.0120 sec 2024-11-21T00:27:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T00:27:24,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:24,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:27:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,785 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T00:27:24,786 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,790 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/recovered.edits] 2024-11-21T00:27:24,792 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67378df27ee845deb024e0fecbf4a7a7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/67378df27ee845deb024e0fecbf4a7a7 2024-11-21T00:27:24,794 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/be3d9ac03aec4a32a2bb9f4e0dcb48d2 2024-11-21T00:27:24,795 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fc4e39af5d784b38b04b6b9cb5317401 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fc4e39af5d784b38b04b6b9cb5317401 2024-11-21T00:27:24,796 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fe01c6578f3a421ea4e1dd94338d58a4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/A/fe01c6578f3a421ea4e1dd94338d58a4 2024-11-21T00:27:24,798 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/383ee51bcf8b4e9bbcb5669b3efc2451 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/383ee51bcf8b4e9bbcb5669b3efc2451 2024-11-21T00:27:24,799 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/9c6420a75e0941308a2335207c023aa9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/9c6420a75e0941308a2335207c023aa9 2024-11-21T00:27:24,800 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/a62f4bc00a4f4e48bda7727008317a23 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/a62f4bc00a4f4e48bda7727008317a23 2024-11-21T00:27:24,801 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/fdd8274e72d646c387932bd743ab94f7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/B/fdd8274e72d646c387932bd743ab94f7 2024-11-21T00:27:24,804 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/4d7dcb2edc834cceb495cd756a2e8897 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/4d7dcb2edc834cceb495cd756a2e8897 2024-11-21T00:27:24,805 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/53da8b8600a24650a0bda03d70192a1b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/53da8b8600a24650a0bda03d70192a1b 2024-11-21T00:27:24,807 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c54f6455030c4dbba5a69f31bf43d3bd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/c54f6455030c4dbba5a69f31bf43d3bd 2024-11-21T00:27:24,808 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d97853d02cef42609d800bb3397e487d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/C/d97853d02cef42609d800bb3397e487d 2024-11-21T00:27:24,810 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/recovered.edits/459.seqid to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e/recovered.edits/459.seqid 2024-11-21T00:27:24,811 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,811 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:27:24,812 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:27:24,813 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T00:27:24,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112107c9470435884daca6e3edf15e1c07a6_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112107c9470435884daca6e3edf15e1c07a6_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,819 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212146587ea8d44994882580a6c996a135_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212146587ea8d44994882580a6c996a135_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126cbcdc8a9a04995b66479db5b480d97_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112126cbcdc8a9a04995b66479db5b480d97_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,822 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ab9d29deed441da91692f17a6c89dfd_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ab9d29deed441da91692f17a6c89dfd_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,823 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213aeea7e0f7a749c89a0b0ad4510f7878_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213aeea7e0f7a749c89a0b0ad4510f7878_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153edade836e5439188c9f645b5731521_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153edade836e5439188c9f645b5731521_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,827 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121569ddaecdd244ed3853ab6d390b16898_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121569ddaecdd244ed3853ab6d390b16898_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,829 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215aa68e11a67e47179b2ae042e283ef8f_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215aa68e11a67e47179b2ae042e283ef8f_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,830 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e8ee12d436045c5b3a8c505865e26fb_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e8ee12d436045c5b3a8c505865e26fb_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,831 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121752bc558853048eaa7678694f9085c9e_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121752bc558853048eaa7678694f9085c9e_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,833 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112178f3fc07a8e9447080d600f07c087b07_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112178f3fc07a8e9447080d600f07c087b07_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,834 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ede51daf0be4a63aaebb53b58c6894a_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217ede51daf0be4a63aaebb53b58c6894a_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121800e3562352d4e719aeeaa248ccebd50_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121800e3562352d4e719aeeaa248ccebd50_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,836 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112191048d1a058442738c943e9c8804ce41_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112191048d1a058442738c943e9c8804ce41_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,837 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121933ea4aaf2c9469aa30da6d96a79b625_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121933ea4aaf2c9469aa30da6d96a79b625_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,839 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a53d926ecb3540e08ece2e473d7ee97b_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a53d926ecb3540e08ece2e473d7ee97b_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,840 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6de63f4ba204d61b69e4af8e8b3c4ea_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6de63f4ba204d61b69e4af8e8b3c4ea_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,841 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121af6c0df9bf0245438b1501e0f43cffde_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121af6c0df9bf0245438b1501e0f43cffde_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,842 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b59d17a3f1e541aa82755a32a06e8c72_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b59d17a3f1e541aa82755a32a06e8c72_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,843 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bc94f7cf0ecc47339fee455da71f62fa_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bc94f7cf0ecc47339fee455da71f62fa_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,844 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd7ec88a74c34667a207073ea9c6bf97_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd7ec88a74c34667a207073ea9c6bf97_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,846 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c22dc84ea6ee4fdc9a86bd725f054514_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c22dc84ea6ee4fdc9a86bd725f054514_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,847 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121da1d20f231d342839484ab80c41b664b_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121da1d20f231d342839484ab80c41b664b_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,848 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dbb23f4c8bbd4a0fa5695dd9ff12e7eb_d3834c10900814f7a41633aca86edf7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dbb23f4c8bbd4a0fa5695dd9ff12e7eb_d3834c10900814f7a41633aca86edf7e 2024-11-21T00:27:24,849 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:27:24,851 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,854 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:27:24,856 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:27:24,857 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,857 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:27:24,857 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148844857"}]},"ts":"9223372036854775807"} 2024-11-21T00:27:24,859 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:27:24,859 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d3834c10900814f7a41633aca86edf7e, NAME => 'TestAcidGuarantees,,1732148816288.d3834c10900814f7a41633aca86edf7e.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:27:24,860 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:27:24,860 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148844860"}]},"ts":"9223372036854775807"} 2024-11-21T00:27:24,862 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:27:24,863 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-21T00:27:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T00:27:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:27:24,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:24,904 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1011473186_22 at /127.0.0.1:34204 [Waiting for operation #140] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-105637966_22 at /127.0.0.1:59174 [Waiting for operation #261] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-105637966_22 at /127.0.0.1:59284 [Waiting for operation #234] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=997 (was 825) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=729 (was 2038) 2024-11-21T00:27:24,916 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=997, ProcessCount=11, AvailableMemoryMB=727 2024-11-21T00:27:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:27:24,917 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:27:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:24,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:27:24,920 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:24,920 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-21T00:27:24,920 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:27:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T00:27:24,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742053_1229 (size=959) 2024-11-21T00:27:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T00:27:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T00:27:25,331 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:27:25,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742054_1230 (size=53) 2024-11-21T00:27:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T00:27:25,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:27:25,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9f0e4f53506bf5b3f736664c2080ec6c, disabling compactions & flushes 2024-11-21T00:27:25,737 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:25,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:25,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. after waiting 0 ms 2024-11-21T00:27:25,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:25,737 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:25,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f0e4f53506bf5b3f736664c2080ec6c: Waiting for close lock at 1732148845737Disabling compacts and flushes for region at 1732148845737Disabling writes for close at 1732148845737Writing region close event to WAL at 1732148845737Closed at 1732148845737 2024-11-21T00:27:25,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:27:25,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148845739"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148845739"}]},"ts":"1732148845739"} 2024-11-21T00:27:25,740 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:27:25,741 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:27:25,742 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148845741"}]},"ts":"1732148845741"} 2024-11-21T00:27:25,743 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:27:25,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, ASSIGN}] 2024-11-21T00:27:25,748 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, ASSIGN 2024-11-21T00:27:25,749 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=false 2024-11-21T00:27:25,900 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=9f0e4f53506bf5b3f736664c2080ec6c, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:25,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, ASSIGN because future has completed 2024-11-21T00:27:25,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:27:26,057 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:26,057 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:27:26,057 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,057 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:27:26,057 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,057 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,059 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,060 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f0e4f53506bf5b3f736664c2080ec6c columnFamilyName A 2024-11-21T00:27:26,060 DEBUG [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:26,060 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] regionserver.HStore(327): Store=9f0e4f53506bf5b3f736664c2080ec6c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:26,061 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,062 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f0e4f53506bf5b3f736664c2080ec6c columnFamilyName B 2024-11-21T00:27:26,062 DEBUG [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:26,062 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] regionserver.HStore(327): Store=9f0e4f53506bf5b3f736664c2080ec6c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T00:27:26,062 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,063 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f0e4f53506bf5b3f736664c2080ec6c columnFamilyName C 2024-11-21T00:27:26,063 DEBUG [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:26,063 INFO [StoreOpener-9f0e4f53506bf5b3f736664c2080ec6c-1 {}] regionserver.HStore(327): Store=9f0e4f53506bf5b3f736664c2080ec6c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:26,064 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,064 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,065 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,065 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,065 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,066 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:27:26,067 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,069 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:27:26,069 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 9f0e4f53506bf5b3f736664c2080ec6c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73457318, jitterRate=0.09459933638572693}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:27:26,069 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:26,070 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 9f0e4f53506bf5b3f736664c2080ec6c: Running coprocessor pre-open hook at 1732148846058Writing region info on filesystem at 1732148846058Initializing all the Stores at 1732148846058Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148846058Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148846058Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148846058Cleaning up temporary data from old regions at 1732148846065 (+7 ms)Running coprocessor post-open hooks at 1732148846069 (+4 ms)Region opened successfully at 1732148846070 (+1 ms) 2024-11-21T00:27:26,070 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., pid=61, masterSystemTime=1732148846054 2024-11-21T00:27:26,072 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:26,072 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:26,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=9f0e4f53506bf5b3f736664c2080ec6c, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:26,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:27:26,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T00:27:26,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 in 172 msec 2024-11-21T00:27:26,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-21T00:27:26,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, ASSIGN in 332 msec 2024-11-21T00:27:26,078 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:27:26,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148846078"}]},"ts":"1732148846078"} 2024-11-21T00:27:26,079 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:27:26,080 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:27:26,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-21T00:27:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-21T00:27:27,072 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:27,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11f0a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,074 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307436fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,076 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e95b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,080 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:27:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:27:27,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c420023 2024-11-21T00:27:27,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:27:27,081 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:27:27,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,084 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13db99c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,085 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f27171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,087 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4c5530, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,088 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2531eee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,090 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d24292b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,091 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64358d6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555bc2ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,099 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f1d67d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7075fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c455a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,105 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa4e7b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3218c6a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,109 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3cdd18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,110 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595ad715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,113 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c575d83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,115 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716d6c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,120 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e3510b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,124 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309a140b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:27,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:27,135 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbe079e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:27,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740302f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:27,145 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,145 DEBUG [Thread-1082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,166 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:27,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:27,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:27,181 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,202 DEBUG [Thread-1078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,211 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,228 DEBUG [Thread-1080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/6103607deb38485c874e3b60841490e6 is 50, key is test_row_0/A:col10/1732148847162/Put/seqid=0 2024-11-21T00:27:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,242 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,252 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-21T00:27:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T00:27:27,266 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:27,266 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:27,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:27,268 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:27,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:27,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148907263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148907264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148907265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148907268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148907271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742055_1231 (size=12001) 2024-11-21T00:27:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T00:27:27,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148907382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148907383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148907383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148907393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148907393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:27,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T00:27:27,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148907592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148907603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148907602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148907603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148907603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/6103607deb38485c874e3b60841490e6 2024-11-21T00:27:27,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:27,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:27,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b6bfa256e4a840f49a88f3447b8c2237 is 50, key is test_row_0/B:col10/1732148847162/Put/seqid=0 2024-11-21T00:27:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742056_1232 (size=12001) 2024-11-21T00:27:27,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:27,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T00:27:27,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148907902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148907914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148907914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148907919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148907919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:28,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:28,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:28,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:28,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:28,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b6bfa256e4a840f49a88f3447b8c2237 2024-11-21T00:27:28,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:28,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/15f9c3bd0905490db107bee7c489c280 is 50, key is test_row_0/C:col10/1732148847162/Put/seqid=0 2024-11-21T00:27:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742057_1233 (size=12001) 2024-11-21T00:27:28,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/15f9c3bd0905490db107bee7c489c280 2024-11-21T00:27:28,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/6103607deb38485c874e3b60841490e6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/6103607deb38485c874e3b60841490e6 2024-11-21T00:27:28,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/6103607deb38485c874e3b60841490e6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:27:28,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b6bfa256e4a840f49a88f3447b8c2237 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b6bfa256e4a840f49a88f3447b8c2237 2024-11-21T00:27:28,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b6bfa256e4a840f49a88f3447b8c2237, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:27:28,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/15f9c3bd0905490db107bee7c489c280 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15f9c3bd0905490db107bee7c489c280 2024-11-21T00:27:28,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15f9c3bd0905490db107bee7c489c280, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:27:28,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9f0e4f53506bf5b3f736664c2080ec6c in 1145ms, sequenceid=13, compaction requested=false 2024-11-21T00:27:28,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:28,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T00:27:28,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:28,357 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T00:27:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/eb7a13a0570541da9713f02631c2d462 is 50, key is test_row_0/A:col10/1732148847234/Put/seqid=0 2024-11-21T00:27:28,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742058_1234 (size=12001) 2024-11-21T00:27:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T00:27:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148908430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148908432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148908432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148908435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148908435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148908547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148908548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148908548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148908753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148908753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148908762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:28,785 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/eb7a13a0570541da9713f02631c2d462 2024-11-21T00:27:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/74bd3703b36b444a86de5f0fa618d19d is 50, key is test_row_0/B:col10/1732148847234/Put/seqid=0 2024-11-21T00:27:28,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742059_1235 (size=12001) 2024-11-21T00:27:28,856 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/74bd3703b36b444a86de5f0fa618d19d 2024-11-21T00:27:28,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/4082fee3919b447a91c81a05f3fa1439 is 50, key is test_row_0/C:col10/1732148847234/Put/seqid=0 2024-11-21T00:27:28,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742060_1236 (size=12001) 2024-11-21T00:27:29,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148909062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148909062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148909072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,144 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:27:29,333 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/4082fee3919b447a91c81a05f3fa1439 2024-11-21T00:27:29,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/eb7a13a0570541da9713f02631c2d462 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/eb7a13a0570541da9713f02631c2d462 2024-11-21T00:27:29,345 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/eb7a13a0570541da9713f02631c2d462, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T00:27:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/74bd3703b36b444a86de5f0fa618d19d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/74bd3703b36b444a86de5f0fa618d19d 2024-11-21T00:27:29,352 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/74bd3703b36b444a86de5f0fa618d19d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T00:27:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/4082fee3919b447a91c81a05f3fa1439 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/4082fee3919b447a91c81a05f3fa1439 2024-11-21T00:27:29,375 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/4082fee3919b447a91c81a05f3fa1439, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T00:27:29,386 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9f0e4f53506bf5b3f736664c2080ec6c in 1028ms, sequenceid=37, compaction requested=false 2024-11-21T00:27:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-21T00:27:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-21T00:27:29,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T00:27:29,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-11-21T00:27:29,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1310 sec 2024-11-21T00:27:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T00:27:29,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:29,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-21T00:27:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:29,424 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:29,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:29,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:29,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:29,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:27:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:29,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7d1b4bf0d5bb49adb178751a287c79b6 is 50, key is test_row_0/A:col10/1732148848433/Put/seqid=0 2024-11-21T00:27:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:29,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742061_1237 (size=16681) 2024-11-21T00:27:29,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148909536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148909546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148909572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148909572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148909582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148909653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148909663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:29,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:29,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:29,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148909862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148909872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:29,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:29,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:29,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:29,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:29,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7d1b4bf0d5bb49adb178751a287c79b6 2024-11-21T00:27:29,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/ee5a060025484e0d816cb23a2a4ebc65 is 50, key is test_row_0/B:col10/1732148848433/Put/seqid=0 2024-11-21T00:27:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:30,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742062_1238 (size=12001) 2024-11-21T00:27:30,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/ee5a060025484e0d816cb23a2a4ebc65 2024-11-21T00:27:30,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/60ac478751b0495b9e00e12d1171325c is 50, key is test_row_0/C:col10/1732148848433/Put/seqid=0 2024-11-21T00:27:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742063_1239 (size=12001) 2024-11-21T00:27:30,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148910172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148910182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/60ac478751b0495b9e00e12d1171325c 2024-11-21T00:27:30,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7d1b4bf0d5bb49adb178751a287c79b6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7d1b4bf0d5bb49adb178751a287c79b6 2024-11-21T00:27:30,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7d1b4bf0d5bb49adb178751a287c79b6, entries=250, sequenceid=51, filesize=16.3 K 2024-11-21T00:27:30,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/ee5a060025484e0d816cb23a2a4ebc65 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ee5a060025484e0d816cb23a2a4ebc65 2024-11-21T00:27:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:30,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ee5a060025484e0d816cb23a2a4ebc65, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:27:30,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/60ac478751b0495b9e00e12d1171325c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/60ac478751b0495b9e00e12d1171325c 2024-11-21T00:27:30,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/60ac478751b0495b9e00e12d1171325c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T00:27:30,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9f0e4f53506bf5b3f736664c2080ec6c in 1110ms, sequenceid=51, compaction requested=true 2024-11-21T00:27:30,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:30,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:30,569 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:30,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:30,569 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:30,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:30,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:30,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:30,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:30,571 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:30,571 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:30,571 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:30,571 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:30,571 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,571 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b6bfa256e4a840f49a88f3447b8c2237, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/74bd3703b36b444a86de5f0fa618d19d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ee5a060025484e0d816cb23a2a4ebc65] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=35.2 K 2024-11-21T00:27:30,571 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,571 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/6103607deb38485c874e3b60841490e6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/eb7a13a0570541da9713f02631c2d462, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7d1b4bf0d5bb49adb178751a287c79b6] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=39.7 K 2024-11-21T00:27:30,572 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b6bfa256e4a840f49a88f3447b8c2237, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148847162 2024-11-21T00:27:30,572 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6103607deb38485c874e3b60841490e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148847162 2024-11-21T00:27:30,572 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 74bd3703b36b444a86de5f0fa618d19d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732148847234 2024-11-21T00:27:30,572 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb7a13a0570541da9713f02631c2d462, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732148847234 2024-11-21T00:27:30,573 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5a060025484e0d816cb23a2a4ebc65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148848433 2024-11-21T00:27:30,573 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d1b4bf0d5bb49adb178751a287c79b6, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148848425 2024-11-21T00:27:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:30,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T00:27:30,598 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:30,599 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/fe14bcd449d9468985d73bc508b61967 is 50, key is test_row_0/B:col10/1732148848433/Put/seqid=0 2024-11-21T00:27:30,607 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:30,608 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/5657256458e24d1890b6eb22cd5b1c7a is 50, key is test_row_0/A:col10/1732148848433/Put/seqid=0 2024-11-21T00:27:30,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/17c83b60c0f3450f8c5ccb9917cd6759 is 50, key is test_row_0/A:col10/1732148849534/Put/seqid=0 2024-11-21T00:27:30,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148910619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148910619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148910621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742064_1240 (size=12104) 2024-11-21T00:27:30,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742066_1242 (size=14341) 2024-11-21T00:27:30,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/17c83b60c0f3450f8c5ccb9917cd6759 2024-11-21T00:27:30,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742065_1241 (size=12104) 2024-11-21T00:27:30,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148910682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148910692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/432c3677ba7d480b98c695730a6938dd is 50, key is test_row_0/B:col10/1732148849534/Put/seqid=0 2024-11-21T00:27:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742067_1243 (size=12001) 2024-11-21T00:27:30,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/432c3677ba7d480b98c695730a6938dd 2024-11-21T00:27:30,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/15b6f0a3cf2f4a5895cf07cf353f0784 is 50, key is test_row_0/C:col10/1732148849534/Put/seqid=0 2024-11-21T00:27:30,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742068_1244 (size=12001) 2024-11-21T00:27:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148910943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148910944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148910952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:30,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:30,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:31,068 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/fe14bcd449d9468985d73bc508b61967 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/fe14bcd449d9468985d73bc508b61967 2024-11-21T00:27:31,074 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into fe14bcd449d9468985d73bc508b61967(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:31,074 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:31,074 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148850569; duration=0sec 2024-11-21T00:27:31,074 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:31,074 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:31,074 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:31,075 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:31,075 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:31,075 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:31,075 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15f9c3bd0905490db107bee7c489c280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/4082fee3919b447a91c81a05f3fa1439, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/60ac478751b0495b9e00e12d1171325c] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=35.2 K 2024-11-21T00:27:31,077 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f9c3bd0905490db107bee7c489c280, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148847162 2024-11-21T00:27:31,077 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4082fee3919b447a91c81a05f3fa1439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732148847234 2024-11-21T00:27:31,077 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ac478751b0495b9e00e12d1171325c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148848433 2024-11-21T00:27:31,091 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:31,091 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/27dcc589326741bcb1aa35b904b928d5 is 50, key is test_row_0/C:col10/1732148848433/Put/seqid=0 2024-11-21T00:27:31,092 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/5657256458e24d1890b6eb22cd5b1c7a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/5657256458e24d1890b6eb22cd5b1c7a 2024-11-21T00:27:31,100 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 5657256458e24d1890b6eb22cd5b1c7a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:31,100 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:31,100 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148850569; duration=0sec 2024-11-21T00:27:31,100 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:31,100 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:31,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742069_1245 (size=12104) 2024-11-21T00:27:31,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:31,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:31,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/15b6f0a3cf2f4a5895cf07cf353f0784 2024-11-21T00:27:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/17c83b60c0f3450f8c5ccb9917cd6759 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/17c83b60c0f3450f8c5ccb9917cd6759 2024-11-21T00:27:31,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/17c83b60c0f3450f8c5ccb9917cd6759, entries=200, sequenceid=74, filesize=14.0 K 2024-11-21T00:27:31,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/432c3677ba7d480b98c695730a6938dd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/432c3677ba7d480b98c695730a6938dd 2024-11-21T00:27:31,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/432c3677ba7d480b98c695730a6938dd, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T00:27:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148911262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148911262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/15b6f0a3cf2f4a5895cf07cf353f0784 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15b6f0a3cf2f4a5895cf07cf353f0784 2024-11-21T00:27:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148911262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15b6f0a3cf2f4a5895cf07cf353f0784, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T00:27:31,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9f0e4f53506bf5b3f736664c2080ec6c in 697ms, sequenceid=74, compaction requested=false 2024-11-21T00:27:31,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:31,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T00:27:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:31,314 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:27:31,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/f09a96897e274e6597ac8b83e80b74d5 is 50, key is test_row_0/A:col10/1732148850591/Put/seqid=0 2024-11-21T00:27:31,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742070_1246 (size=12001) 2024-11-21T00:27:31,357 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/f09a96897e274e6597ac8b83e80b74d5 2024-11-21T00:27:31,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/4605189d023a40e2b32e8747b2788a91 is 50, key is test_row_0/B:col10/1732148850591/Put/seqid=0 2024-11-21T00:27:31,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742071_1247 (size=12001) 2024-11-21T00:27:31,410 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/4605189d023a40e2b32e8747b2788a91 2024-11-21T00:27:31,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/763d1daa6ea84b7c8278f7566c0d813d is 50, key is test_row_0/C:col10/1732148850591/Put/seqid=0 2024-11-21T00:27:31,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742072_1248 (size=12001) 2024-11-21T00:27:31,523 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/763d1daa6ea84b7c8278f7566c0d813d 2024-11-21T00:27:31,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/f09a96897e274e6597ac8b83e80b74d5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f09a96897e274e6597ac8b83e80b74d5 2024-11-21T00:27:31,540 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f09a96897e274e6597ac8b83e80b74d5, entries=150, sequenceid=89, filesize=11.7 K 2024-11-21T00:27:31,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/4605189d023a40e2b32e8747b2788a91 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/4605189d023a40e2b32e8747b2788a91 2024-11-21T00:27:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:31,567 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/27dcc589326741bcb1aa35b904b928d5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/27dcc589326741bcb1aa35b904b928d5 2024-11-21T00:27:31,574 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/4605189d023a40e2b32e8747b2788a91, entries=150, sequenceid=89, filesize=11.7 K 2024-11-21T00:27:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/763d1daa6ea84b7c8278f7566c0d813d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/763d1daa6ea84b7c8278f7566c0d813d 2024-11-21T00:27:31,606 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into 27dcc589326741bcb1aa35b904b928d5(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:31,606 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:31,606 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148850570; duration=0sec 2024-11-21T00:27:31,606 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:31,606 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:31,615 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/763d1daa6ea84b7c8278f7566c0d813d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-21T00:27:31,621 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 9f0e4f53506bf5b3f736664c2080ec6c in 307ms, sequenceid=89, compaction requested=true 2024-11-21T00:27:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-21T00:27:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-21T00:27:31,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T00:27:31,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-11-21T00:27:31,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.2150 sec 2024-11-21T00:27:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:31,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:31,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:31,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/3d99332134cc41b7858779a41c12f2a6 is 50, key is test_row_0/A:col10/1732148851741/Put/seqid=0 2024-11-21T00:27:31,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742073_1249 (size=12001) 2024-11-21T00:27:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148911800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148911802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148911810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148911812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148911813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148911923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148911923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148911923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148911923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148911923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/3d99332134cc41b7858779a41c12f2a6 2024-11-21T00:27:32,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/d83489c4371149d281d88f91f0966e93 is 50, key is test_row_0/B:col10/1732148851741/Put/seqid=0 2024-11-21T00:27:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742074_1250 (size=12001) 2024-11-21T00:27:32,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/d83489c4371149d281d88f91f0966e93 2024-11-21T00:27:32,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/43a2280b481a4f3c99986a737be46757 is 50, key is test_row_0/C:col10/1732148851741/Put/seqid=0 2024-11-21T00:27:32,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742075_1251 (size=12001) 2024-11-21T00:27:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148912442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148912442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148912442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148912442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148912442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/43a2280b481a4f3c99986a737be46757 2024-11-21T00:27:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/3d99332134cc41b7858779a41c12f2a6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3d99332134cc41b7858779a41c12f2a6 2024-11-21T00:27:32,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3d99332134cc41b7858779a41c12f2a6, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T00:27:32,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/d83489c4371149d281d88f91f0966e93 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d83489c4371149d281d88f91f0966e93 2024-11-21T00:27:32,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d83489c4371149d281d88f91f0966e93, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T00:27:32,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/43a2280b481a4f3c99986a737be46757 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/43a2280b481a4f3c99986a737be46757 2024-11-21T00:27:32,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/43a2280b481a4f3c99986a737be46757, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T00:27:32,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9f0e4f53506bf5b3f736664c2080ec6c in 1037ms, sequenceid=101, compaction requested=true 2024-11-21T00:27:32,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:32,781 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:32,781 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:32,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:32,782 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:32,782 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:32,782 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:32,783 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/fe14bcd449d9468985d73bc508b61967, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/432c3677ba7d480b98c695730a6938dd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/4605189d023a40e2b32e8747b2788a91, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d83489c4371149d281d88f91f0966e93] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=47.0 K 2024-11-21T00:27:32,783 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting fe14bcd449d9468985d73bc508b61967, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148848433 2024-11-21T00:27:32,783 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:32,783 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:32,783 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 432c3677ba7d480b98c695730a6938dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732148849534 2024-11-21T00:27:32,783 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:32,783 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/5657256458e24d1890b6eb22cd5b1c7a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/17c83b60c0f3450f8c5ccb9917cd6759, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f09a96897e274e6597ac8b83e80b74d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3d99332134cc41b7858779a41c12f2a6] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=49.3 K 2024-11-21T00:27:32,784 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4605189d023a40e2b32e8747b2788a91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732148850591 2024-11-21T00:27:32,784 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5657256458e24d1890b6eb22cd5b1c7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148848433 2024-11-21T00:27:32,785 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c83b60c0f3450f8c5ccb9917cd6759, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732148849526 2024-11-21T00:27:32,785 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d83489c4371149d281d88f91f0966e93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732148851715 2024-11-21T00:27:32,786 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f09a96897e274e6597ac8b83e80b74d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732148850591 2024-11-21T00:27:32,787 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d99332134cc41b7858779a41c12f2a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732148851715 2024-11-21T00:27:32,814 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:32,815 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/a4d2d65df3fa4082ab875b184315af3c is 50, key is test_row_0/B:col10/1732148851741/Put/seqid=0 2024-11-21T00:27:32,822 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:32,823 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/2a7ec4588b5143acb09a71cff5f68eaf is 50, key is test_row_0/A:col10/1732148851741/Put/seqid=0 2024-11-21T00:27:32,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742076_1252 (size=12241) 2024-11-21T00:27:32,875 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/a4d2d65df3fa4082ab875b184315af3c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a4d2d65df3fa4082ab875b184315af3c 2024-11-21T00:27:32,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742077_1253 (size=12241) 2024-11-21T00:27:32,884 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into a4d2d65df3fa4082ab875b184315af3c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:32,884 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:32,884 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=12, startTime=1732148852781; duration=0sec 2024-11-21T00:27:32,885 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:32,885 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:32,885 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:32,886 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:32,886 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:32,886 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:32,886 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/27dcc589326741bcb1aa35b904b928d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15b6f0a3cf2f4a5895cf07cf353f0784, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/763d1daa6ea84b7c8278f7566c0d813d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/43a2280b481a4f3c99986a737be46757] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=47.0 K 2024-11-21T00:27:32,887 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 27dcc589326741bcb1aa35b904b928d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732148848433 2024-11-21T00:27:32,887 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b6f0a3cf2f4a5895cf07cf353f0784, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732148849534 2024-11-21T00:27:32,888 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 763d1daa6ea84b7c8278f7566c0d813d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732148850591 2024-11-21T00:27:32,888 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a2280b481a4f3c99986a737be46757, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732148851715 2024-11-21T00:27:32,911 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:32,912 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/cfc39461ad82473e878d287c9571c107 is 50, key is test_row_0/C:col10/1732148851741/Put/seqid=0 2024-11-21T00:27:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742078_1254 (size=12241) 2024-11-21T00:27:32,955 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/cfc39461ad82473e878d287c9571c107 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cfc39461ad82473e878d287c9571c107 2024-11-21T00:27:32,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:32,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T00:27:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:32,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/9492a85facbf457f8149e30ff698f6ab is 50, key is test_row_0/A:col10/1732148852955/Put/seqid=0 2024-11-21T00:27:32,971 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into cfc39461ad82473e878d287c9571c107(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:32,971 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:32,971 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=12, startTime=1732148852781; duration=0sec 2024-11-21T00:27:32,971 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:32,971 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148912968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148912969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148912969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148912970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148912971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:32,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742079_1255 (size=16781) 2024-11-21T00:27:32,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/9492a85facbf457f8149e30ff698f6ab 2024-11-21T00:27:33,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/40b244f38af34b8f9ac03f16b59d94c9 is 50, key is test_row_0/B:col10/1732148852955/Put/seqid=0 2024-11-21T00:27:33,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742080_1256 (size=12101) 2024-11-21T00:27:33,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/40b244f38af34b8f9ac03f16b59d94c9 2024-11-21T00:27:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148913082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148913082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148913082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148913082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148913082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/cdc5ee13f0b64ccf978866f5300df3dd is 50, key is test_row_0/C:col10/1732148852955/Put/seqid=0 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742081_1257 (size=12101) 2024-11-21T00:27:33,295 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/2a7ec4588b5143acb09a71cff5f68eaf as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2a7ec4588b5143acb09a71cff5f68eaf 2024-11-21T00:27:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148913293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148913293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148913293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148913293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148913292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,306 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 2a7ec4588b5143acb09a71cff5f68eaf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:33,306 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:33,306 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=12, startTime=1732148852780; duration=0sec 2024-11-21T00:27:33,306 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:33,306 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:33,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/cdc5ee13f0b64ccf978866f5300df3dd 2024-11-21T00:27:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/9492a85facbf457f8149e30ff698f6ab as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9492a85facbf457f8149e30ff698f6ab 2024-11-21T00:27:33,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9492a85facbf457f8149e30ff698f6ab, entries=250, sequenceid=130, filesize=16.4 K 2024-11-21T00:27:33,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/40b244f38af34b8f9ac03f16b59d94c9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/40b244f38af34b8f9ac03f16b59d94c9 2024-11-21T00:27:33,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/40b244f38af34b8f9ac03f16b59d94c9, entries=150, sequenceid=130, filesize=11.8 K 2024-11-21T00:27:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/cdc5ee13f0b64ccf978866f5300df3dd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cdc5ee13f0b64ccf978866f5300df3dd 2024-11-21T00:27:33,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cdc5ee13f0b64ccf978866f5300df3dd, entries=150, sequenceid=130, filesize=11.8 K 2024-11-21T00:27:33,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9f0e4f53506bf5b3f736664c2080ec6c in 609ms, sequenceid=130, compaction requested=false 2024-11-21T00:27:33,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T00:27:33,583 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:33,583 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-21T00:27:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:33,585 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:33,586 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:33,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:33,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:33,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:33,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/bb2f8378546e40c983fceff830965917 is 50, key is test_row_0/A:col10/1732148852968/Put/seqid=0 2024-11-21T00:27:33,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742082_1258 (size=12151) 2024-11-21T00:27:33,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148913655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148913656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148913661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148913664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148913686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:33,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:33,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:33,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:33,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148913773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148913773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148913773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148913783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148913805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:33,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:33,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148913984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148913993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148913993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148913993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148914028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/bb2f8378546e40c983fceff830965917 2024-11-21T00:27:34,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/6e56360c6c5d408788a3824de45db512 is 50, key is test_row_0/B:col10/1732148852968/Put/seqid=0 2024-11-21T00:27:34,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742083_1259 (size=12151) 2024-11-21T00:27:34,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/6e56360c6c5d408788a3824de45db512 2024-11-21T00:27:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:34,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/12e1b94eced3456ab43407ad3d372689 is 50, key is test_row_0/C:col10/1732148852968/Put/seqid=0 2024-11-21T00:27:34,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742084_1260 (size=12151) 2024-11-21T00:27:34,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/12e1b94eced3456ab43407ad3d372689 2024-11-21T00:27:34,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/bb2f8378546e40c983fceff830965917 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb2f8378546e40c983fceff830965917 2024-11-21T00:27:34,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb2f8378546e40c983fceff830965917, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T00:27:34,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/6e56360c6c5d408788a3824de45db512 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/6e56360c6c5d408788a3824de45db512 2024-11-21T00:27:34,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/6e56360c6c5d408788a3824de45db512, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T00:27:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/12e1b94eced3456ab43407ad3d372689 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/12e1b94eced3456ab43407ad3d372689 2024-11-21T00:27:34,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148914302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148914312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148914313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148914313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/12e1b94eced3456ab43407ad3d372689, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T00:27:34,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9f0e4f53506bf5b3f736664c2080ec6c in 713ms, sequenceid=143, compaction requested=true 2024-11-21T00:27:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:34,319 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:34,319 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:34,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:34,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:34,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:34,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:34,320 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:34,320 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:34,321 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,321 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a4d2d65df3fa4082ab875b184315af3c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/40b244f38af34b8f9ac03f16b59d94c9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/6e56360c6c5d408788a3824de45db512] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=35.6 K 2024-11-21T00:27:34,321 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:34,321 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:34,321 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,321 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d2d65df3fa4082ab875b184315af3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732148851715 2024-11-21T00:27:34,321 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2a7ec4588b5143acb09a71cff5f68eaf, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9492a85facbf457f8149e30ff698f6ab, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb2f8378546e40c983fceff830965917] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=40.2 K 2024-11-21T00:27:34,322 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b244f38af34b8f9ac03f16b59d94c9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732148851800 2024-11-21T00:27:34,322 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a7ec4588b5143acb09a71cff5f68eaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732148851715 2024-11-21T00:27:34,322 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e56360c6c5d408788a3824de45db512, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732148852968 2024-11-21T00:27:34,322 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9492a85facbf457f8149e30ff698f6ab, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732148851800 2024-11-21T00:27:34,323 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2f8378546e40c983fceff830965917, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732148852968 2024-11-21T00:27:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:34,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:34,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:27:34,365 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:34,366 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/887c7b3dad9c4ed1a21f1d6507f9354a is 50, key is test_row_0/A:col10/1732148852968/Put/seqid=0 2024-11-21T00:27:34,376 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:34,376 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/0f872ac07ee940f5ad1bde9efcf17ad3 is 50, key is test_row_0/B:col10/1732148852968/Put/seqid=0 2024-11-21T00:27:34,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/d3f2f50decae43f0a77f6c1b2db23c8a is 50, key is test_row_0/A:col10/1732148853650/Put/seqid=0 2024-11-21T00:27:34,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148914395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742085_1261 (size=12493) 2024-11-21T00:27:34,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742087_1263 (size=14541) 2024-11-21T00:27:34,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742086_1262 (size=12493) 2024-11-21T00:27:34,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148914512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148914723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148914814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148914822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148914822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148914822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:34,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:34,857 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/0f872ac07ee940f5ad1bde9efcf17ad3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0f872ac07ee940f5ad1bde9efcf17ad3 2024-11-21T00:27:34,858 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/887c7b3dad9c4ed1a21f1d6507f9354a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/887c7b3dad9c4ed1a21f1d6507f9354a 2024-11-21T00:27:34,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/d3f2f50decae43f0a77f6c1b2db23c8a 2024-11-21T00:27:34,879 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 887c7b3dad9c4ed1a21f1d6507f9354a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:34,879 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:34,879 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148854319; duration=0sec 2024-11-21T00:27:34,879 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:34,880 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:34,880 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:34,880 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into 0f872ac07ee940f5ad1bde9efcf17ad3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:34,880 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:34,880 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148854319; duration=0sec 2024-11-21T00:27:34,881 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:34,881 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:34,889 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:34,889 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:34,889 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:34,889 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cfc39461ad82473e878d287c9571c107, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cdc5ee13f0b64ccf978866f5300df3dd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/12e1b94eced3456ab43407ad3d372689] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=35.6 K 2024-11-21T00:27:34,892 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc39461ad82473e878d287c9571c107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732148851715 2024-11-21T00:27:34,894 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc5ee13f0b64ccf978866f5300df3dd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732148851800 2024-11-21T00:27:34,894 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e1b94eced3456ab43407ad3d372689, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732148852968 2024-11-21T00:27:34,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/3298037bc5de404882f0709825ec5c4e is 50, key is test_row_0/B:col10/1732148853650/Put/seqid=0 2024-11-21T00:27:34,942 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:34,943 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/c0d647b1acdf4cbea6dbd86645d27e59 is 50, key is test_row_0/C:col10/1732148852968/Put/seqid=0 2024-11-21T00:27:35,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742089_1265 (size=12493) 2024-11-21T00:27:35,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742088_1264 (size=12151) 2024-11-21T00:27:35,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/3298037bc5de404882f0709825ec5c4e 2024-11-21T00:27:35,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/a7af240c11524c96ad00db0089a1b74f is 50, key is test_row_0/C:col10/1732148853650/Put/seqid=0 2024-11-21T00:27:35,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148915032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:35,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742090_1266 (size=12151) 2024-11-21T00:27:35,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/c0d647b1acdf4cbea6dbd86645d27e59 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0d647b1acdf4cbea6dbd86645d27e59 2024-11-21T00:27:35,448 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into c0d647b1acdf4cbea6dbd86645d27e59(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:35,448 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:35,448 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148854320; duration=0sec 2024-11-21T00:27:35,448 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:35,448 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:35,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:35,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/a7af240c11524c96ad00db0089a1b74f 2024-11-21T00:27:35,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/d3f2f50decae43f0a77f6c1b2db23c8a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/d3f2f50decae43f0a77f6c1b2db23c8a 2024-11-21T00:27:35,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/d3f2f50decae43f0a77f6c1b2db23c8a, entries=200, sequenceid=170, filesize=14.2 K 2024-11-21T00:27:35,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/3298037bc5de404882f0709825ec5c4e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/3298037bc5de404882f0709825ec5c4e 2024-11-21T00:27:35,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/3298037bc5de404882f0709825ec5c4e, entries=150, sequenceid=170, filesize=11.9 K 2024-11-21T00:27:35,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/a7af240c11524c96ad00db0089a1b74f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a7af240c11524c96ad00db0089a1b74f 2024-11-21T00:27:35,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148915553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:35,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a7af240c11524c96ad00db0089a1b74f, entries=150, sequenceid=170, filesize=11.9 K 2024-11-21T00:27:35,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9f0e4f53506bf5b3f736664c2080ec6c in 1214ms, sequenceid=170, compaction requested=false 2024-11-21T00:27:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:35,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T00:27:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,639 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T00:27:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/9cbe6c6c119943608cf9068644b112ef is 50, key is test_row_0/A:col10/1732148854387/Put/seqid=0 2024-11-21T00:27:35,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742091_1267 (size=12151) 2024-11-21T00:27:35,695 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/9cbe6c6c119943608cf9068644b112ef 2024-11-21T00:27:35,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/7bcb90468cf74d009cc53a989cb4ad01 is 50, key is test_row_0/B:col10/1732148854387/Put/seqid=0 2024-11-21T00:27:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:35,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742092_1268 (size=12151) 2024-11-21T00:27:35,742 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/7bcb90468cf74d009cc53a989cb4ad01 2024-11-21T00:27:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/7f928af45184487cb36cd91256a5b1d9 is 50, key is test_row_0/C:col10/1732148854387/Put/seqid=0 2024-11-21T00:27:35,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742093_1269 (size=12151) 2024-11-21T00:27:35,814 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/7f928af45184487cb36cd91256a5b1d9 2024-11-21T00:27:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/9cbe6c6c119943608cf9068644b112ef as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9cbe6c6c119943608cf9068644b112ef 2024-11-21T00:27:35,825 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9cbe6c6c119943608cf9068644b112ef, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T00:27:35,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/7bcb90468cf74d009cc53a989cb4ad01 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/7bcb90468cf74d009cc53a989cb4ad01 2024-11-21T00:27:35,834 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/7bcb90468cf74d009cc53a989cb4ad01, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T00:27:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/7f928af45184487cb36cd91256a5b1d9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7f928af45184487cb36cd91256a5b1d9 2024-11-21T00:27:35,843 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7f928af45184487cb36cd91256a5b1d9, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T00:27:35,845 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 9f0e4f53506bf5b3f736664c2080ec6c in 206ms, sequenceid=182, compaction requested=true 2024-11-21T00:27:35,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:35,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:35,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-21T00:27:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-21T00:27:35,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T00:27:35,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2640 sec 2024-11-21T00:27:35,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2710 sec 2024-11-21T00:27:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:35,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:35,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7a07fca49d2a42feb789ed179032517e is 50, key is test_row_0/A:col10/1732148855871/Put/seqid=0 2024-11-21T00:27:35,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742094_1270 (size=9757) 2024-11-21T00:27:35,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7a07fca49d2a42feb789ed179032517e 2024-11-21T00:27:35,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/f570c8fd1f124a0ea6ebd04452252f02 is 50, key is test_row_0/B:col10/1732148855871/Put/seqid=0 2024-11-21T00:27:36,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148915984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148915990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148915991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148916002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742095_1271 (size=9757) 2024-11-21T00:27:36,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/f570c8fd1f124a0ea6ebd04452252f02 2024-11-21T00:27:36,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/af4efc562bb04d80bd114f33198b87b9 is 50, key is test_row_0/C:col10/1732148855871/Put/seqid=0 2024-11-21T00:27:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742096_1272 (size=9757) 2024-11-21T00:27:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148916113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148916114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148916114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148916123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148916332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148916332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148916332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148916333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/af4efc562bb04d80bd114f33198b87b9 2024-11-21T00:27:36,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7a07fca49d2a42feb789ed179032517e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7a07fca49d2a42feb789ed179032517e 2024-11-21T00:27:36,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7a07fca49d2a42feb789ed179032517e, entries=100, sequenceid=193, filesize=9.5 K 2024-11-21T00:27:36,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/f570c8fd1f124a0ea6ebd04452252f02 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f570c8fd1f124a0ea6ebd04452252f02 2024-11-21T00:27:36,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f570c8fd1f124a0ea6ebd04452252f02, entries=100, sequenceid=193, filesize=9.5 K 2024-11-21T00:27:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/af4efc562bb04d80bd114f33198b87b9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/af4efc562bb04d80bd114f33198b87b9 2024-11-21T00:27:36,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/af4efc562bb04d80bd114f33198b87b9, entries=100, sequenceid=193, filesize=9.5 K 2024-11-21T00:27:36,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9f0e4f53506bf5b3f736664c2080ec6c in 660ms, sequenceid=193, compaction requested=true 2024-11-21T00:27:36,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:36,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:36,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:36,532 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:36,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:36,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:36,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:36,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:27:36,533 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:36,542 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48942 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:36,542 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:36,542 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:36,542 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/887c7b3dad9c4ed1a21f1d6507f9354a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/d3f2f50decae43f0a77f6c1b2db23c8a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9cbe6c6c119943608cf9068644b112ef, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7a07fca49d2a42feb789ed179032517e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=47.8 K 2024-11-21T00:27:36,550 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887c7b3dad9c4ed1a21f1d6507f9354a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732148852968 2024-11-21T00:27:36,553 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f2f50decae43f0a77f6c1b2db23c8a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732148853650 2024-11-21T00:27:36,553 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbe6c6c119943608cf9068644b112ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732148854366 2024-11-21T00:27:36,555 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a07fca49d2a42feb789ed179032517e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732148855866 2024-11-21T00:27:36,558 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:36,558 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:36,558 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:36,558 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0f872ac07ee940f5ad1bde9efcf17ad3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/3298037bc5de404882f0709825ec5c4e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/7bcb90468cf74d009cc53a989cb4ad01, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f570c8fd1f124a0ea6ebd04452252f02] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=45.5 K 2024-11-21T00:27:36,561 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f872ac07ee940f5ad1bde9efcf17ad3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732148852968 2024-11-21T00:27:36,561 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3298037bc5de404882f0709825ec5c4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732148853650 2024-11-21T00:27:36,566 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bcb90468cf74d009cc53a989cb4ad01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732148854366 2024-11-21T00:27:36,567 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f570c8fd1f124a0ea6ebd04452252f02, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732148855866 2024-11-21T00:27:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:36,590 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:36,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:27:36,601 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:36,601 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/8268e028b0f34839b3f148ed65253a5b is 50, key is test_row_0/A:col10/1732148855871/Put/seqid=0 2024-11-21T00:27:36,610 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:36,611 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/a68db8825c4c4391a202df398cab44f5 is 50, key is test_row_0/B:col10/1732148855871/Put/seqid=0 2024-11-21T00:27:36,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/e8300da868d54c4ba8e6215af0ffcb26 is 50, key is test_row_0/A:col10/1732148855989/Put/seqid=0 2024-11-21T00:27:36,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148916642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148916643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148916643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148916645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148916647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742097_1273 (size=12629) 2024-11-21T00:27:36,681 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/8268e028b0f34839b3f148ed65253a5b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8268e028b0f34839b3f148ed65253a5b 2024-11-21T00:27:36,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742098_1274 (size=12629) 2024-11-21T00:27:36,695 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 8268e028b0f34839b3f148ed65253a5b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:36,695 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:36,695 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=12, startTime=1732148856532; duration=0sec 2024-11-21T00:27:36,695 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:36,695 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:36,695 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:36,697 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:36,697 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:36,697 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:36,698 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0d647b1acdf4cbea6dbd86645d27e59, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a7af240c11524c96ad00db0089a1b74f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7f928af45184487cb36cd91256a5b1d9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/af4efc562bb04d80bd114f33198b87b9] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=45.5 K 2024-11-21T00:27:36,698 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d647b1acdf4cbea6dbd86645d27e59, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732148852968 2024-11-21T00:27:36,698 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/a68db8825c4c4391a202df398cab44f5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a68db8825c4c4391a202df398cab44f5 2024-11-21T00:27:36,699 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7af240c11524c96ad00db0089a1b74f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732148853650 2024-11-21T00:27:36,700 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f928af45184487cb36cd91256a5b1d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732148854366 2024-11-21T00:27:36,700 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4efc562bb04d80bd114f33198b87b9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732148855866 2024-11-21T00:27:36,706 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into a68db8825c4c4391a202df398cab44f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:36,706 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:36,706 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=12, startTime=1732148856532; duration=0sec 2024-11-21T00:27:36,706 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:36,706 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:36,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742099_1275 (size=12151) 2024-11-21T00:27:36,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/e8300da868d54c4ba8e6215af0ffcb26 2024-11-21T00:27:36,724 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:36,725 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/9022793c0fe445b4b80d829265ef0d91 is 50, key is test_row_0/C:col10/1732148855871/Put/seqid=0 2024-11-21T00:27:36,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9fb0c3701baf468da6cd1616b40d92d3 is 50, key is test_row_0/B:col10/1732148855989/Put/seqid=0 2024-11-21T00:27:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148916765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148916765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742100_1276 (size=12629) 2024-11-21T00:27:36,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742101_1277 (size=12151) 2024-11-21T00:27:36,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9fb0c3701baf468da6cd1616b40d92d3 2024-11-21T00:27:36,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/b67d266f09e24bd0ad8d03e056c8ebcb is 50, key is test_row_0/C:col10/1732148855989/Put/seqid=0 2024-11-21T00:27:36,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742102_1278 (size=12151) 2024-11-21T00:27:36,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148916982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148916983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148917152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148917152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148917168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,198 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/9022793c0fe445b4b80d829265ef0d91 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9022793c0fe445b4b80d829265ef0d91 2024-11-21T00:27:37,210 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into 9022793c0fe445b4b80d829265ef0d91(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:37,210 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:37,211 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=12, startTime=1732148856533; duration=0sec 2024-11-21T00:27:37,211 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:37,211 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:37,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/b67d266f09e24bd0ad8d03e056c8ebcb 2024-11-21T00:27:37,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/e8300da868d54c4ba8e6215af0ffcb26 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/e8300da868d54c4ba8e6215af0ffcb26 2024-11-21T00:27:37,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148917293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148917294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/e8300da868d54c4ba8e6215af0ffcb26, entries=150, sequenceid=219, filesize=11.9 K 2024-11-21T00:27:37,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9fb0c3701baf468da6cd1616b40d92d3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9fb0c3701baf468da6cd1616b40d92d3 2024-11-21T00:27:37,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9fb0c3701baf468da6cd1616b40d92d3, entries=150, sequenceid=219, filesize=11.9 K 2024-11-21T00:27:37,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/b67d266f09e24bd0ad8d03e056c8ebcb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b67d266f09e24bd0ad8d03e056c8ebcb 2024-11-21T00:27:37,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b67d266f09e24bd0ad8d03e056c8ebcb, entries=150, sequenceid=219, filesize=11.9 K 2024-11-21T00:27:37,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9f0e4f53506bf5b3f736664c2080ec6c in 735ms, sequenceid=219, compaction requested=false 2024-11-21T00:27:37,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T00:27:37,752 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:37,753 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T00:27:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:27:37,756 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:37,758 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:37,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:37,806 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:37,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:37,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/58060e1ef1b14371bb654d0c49ea562f is 50, key is test_row_0/A:col10/1732148856643/Put/seqid=0 2024-11-21T00:27:37,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742103_1279 (size=14541) 2024-11-21T00:27:37,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/58060e1ef1b14371bb654d0c49ea562f 2024-11-21T00:27:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:27:37,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/bda0e6f5fb7d4a068a2e5065e66f1608 is 50, key is test_row_0/B:col10/1732148856643/Put/seqid=0 2024-11-21T00:27:37,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742104_1280 (size=12151) 2024-11-21T00:27:37,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/bda0e6f5fb7d4a068a2e5065e66f1608 2024-11-21T00:27:37,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:27:37,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:37,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:37,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:37,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:37,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:37,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148917915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148917914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:37,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/a4097c4bed4e4c2b94929e0511e49586 is 50, key is test_row_0/C:col10/1732148856643/Put/seqid=0 2024-11-21T00:27:37,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742105_1281 (size=12151) 2024-11-21T00:27:37,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/a4097c4bed4e4c2b94929e0511e49586 2024-11-21T00:27:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/58060e1ef1b14371bb654d0c49ea562f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/58060e1ef1b14371bb654d0c49ea562f 2024-11-21T00:27:37,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/58060e1ef1b14371bb654d0c49ea562f, entries=200, sequenceid=233, filesize=14.2 K 2024-11-21T00:27:37,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/bda0e6f5fb7d4a068a2e5065e66f1608 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/bda0e6f5fb7d4a068a2e5065e66f1608 2024-11-21T00:27:37,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/bda0e6f5fb7d4a068a2e5065e66f1608, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T00:27:37,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/a4097c4bed4e4c2b94929e0511e49586 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a4097c4bed4e4c2b94929e0511e49586 2024-11-21T00:27:37,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a4097c4bed4e4c2b94929e0511e49586, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T00:27:37,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9f0e4f53506bf5b3f736664c2080ec6c in 189ms, sequenceid=233, compaction requested=true 2024-11-21T00:27:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:37,996 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:37,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:37,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:37,997 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:37,997 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:37,997 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:37,997 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:37,997 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8268e028b0f34839b3f148ed65253a5b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/e8300da868d54c4ba8e6215af0ffcb26, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/58060e1ef1b14371bb654d0c49ea562f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=38.4 K 2024-11-21T00:27:37,998 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8268e028b0f34839b3f148ed65253a5b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732148854393 2024-11-21T00:27:37,999 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:37,999 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:37,999 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:37,999 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a68db8825c4c4391a202df398cab44f5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9fb0c3701baf468da6cd1616b40d92d3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/bda0e6f5fb7d4a068a2e5065e66f1608] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=36.1 K 2024-11-21T00:27:37,999 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8300da868d54c4ba8e6215af0ffcb26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732148855949 2024-11-21T00:27:38,000 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58060e1ef1b14371bb654d0c49ea562f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732148856606 2024-11-21T00:27:38,000 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a68db8825c4c4391a202df398cab44f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732148854393 2024-11-21T00:27:38,000 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb0c3701baf468da6cd1616b40d92d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732148855949 2024-11-21T00:27:38,001 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bda0e6f5fb7d4a068a2e5065e66f1608, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732148856643 2024-11-21T00:27:38,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:38,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:38,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:38,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:38,010 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:38,010 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/839b0ee9018c4a40b448351f8aeccfe6 is 50, key is test_row_0/A:col10/1732148856643/Put/seqid=0 2024-11-21T00:27:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:38,035 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:38,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:27:38,042 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:38,043 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/be27d92fe7fe415abea3f2cc2f8e2182 is 50, key is test_row_0/B:col10/1732148856643/Put/seqid=0 2024-11-21T00:27:38,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/8a4380a6b4c849318ed0cce99e7edba0 is 50, key is test_row_0/A:col10/1732148857903/Put/seqid=0 2024-11-21T00:27:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:27:38,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:27:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:38,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:38,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742107_1283 (size=12731) 2024-11-21T00:27:38,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742106_1282 (size=12731) 2024-11-21T00:27:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148918102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,115 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/839b0ee9018c4a40b448351f8aeccfe6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/839b0ee9018c4a40b448351f8aeccfe6 2024-11-21T00:27:38,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148918112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,125 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 839b0ee9018c4a40b448351f8aeccfe6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:38,125 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:38,125 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148857996; duration=0sec 2024-11-21T00:27:38,125 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:38,125 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:38,125 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:38,126 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:38,126 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:38,126 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:38,126 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9022793c0fe445b4b80d829265ef0d91, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b67d266f09e24bd0ad8d03e056c8ebcb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a4097c4bed4e4c2b94929e0511e49586] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=36.1 K 2024-11-21T00:27:38,126 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9022793c0fe445b4b80d829265ef0d91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732148854393 2024-11-21T00:27:38,127 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b67d266f09e24bd0ad8d03e056c8ebcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732148855949 2024-11-21T00:27:38,127 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4097c4bed4e4c2b94929e0511e49586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732148856643 2024-11-21T00:27:38,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742108_1284 (size=12251) 2024-11-21T00:27:38,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/8a4380a6b4c849318ed0cce99e7edba0 2024-11-21T00:27:38,146 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:38,147 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f0ed2f03b46e4796b855d4d774dde8d4 is 50, key is test_row_0/C:col10/1732148856643/Put/seqid=0 2024-11-21T00:27:38,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/dd54e5bc58e0401884121d303b664e9b is 50, key is test_row_0/B:col10/1732148857903/Put/seqid=0 2024-11-21T00:27:38,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148918164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,172 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148918172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,182 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742109_1285 (size=12731) 2024-11-21T00:27:38,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742110_1286 (size=12251) 2024-11-21T00:27:38,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/dd54e5bc58e0401884121d303b664e9b 2024-11-21T00:27:38,198 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f0ed2f03b46e4796b855d4d774dde8d4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f0ed2f03b46e4796b855d4d774dde8d4 2024-11-21T00:27:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148918192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,199 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,204 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into f0ed2f03b46e4796b855d4d774dde8d4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:38,205 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:38,205 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148858008; duration=0sec 2024-11-21T00:27:38,205 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:38,205 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:38,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/fe9d9799c0954309ac1713a7de0b57aa is 50, key is test_row_0/C:col10/1732148857903/Put/seqid=0 2024-11-21T00:27:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148918222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148918222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:27:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:38,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:38,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742111_1287 (size=12251) 2024-11-21T00:27:38,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/fe9d9799c0954309ac1713a7de0b57aa 2024-11-21T00:27:38,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/8a4380a6b4c849318ed0cce99e7edba0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8a4380a6b4c849318ed0cce99e7edba0 2024-11-21T00:27:38,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8a4380a6b4c849318ed0cce99e7edba0, entries=150, sequenceid=260, filesize=12.0 K 2024-11-21T00:27:38,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/dd54e5bc58e0401884121d303b664e9b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/dd54e5bc58e0401884121d303b664e9b 2024-11-21T00:27:38,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/dd54e5bc58e0401884121d303b664e9b, entries=150, sequenceid=260, filesize=12.0 K 2024-11-21T00:27:38,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/fe9d9799c0954309ac1713a7de0b57aa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/fe9d9799c0954309ac1713a7de0b57aa 2024-11-21T00:27:38,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/fe9d9799c0954309ac1713a7de0b57aa, entries=150, sequenceid=260, filesize=12.0 K 2024-11-21T00:27:38,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9f0e4f53506bf5b3f736664c2080ec6c in 268ms, sequenceid=260, compaction requested=false 2024-11-21T00:27:38,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:27:38,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T00:27:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:38,401 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T00:27:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/b6d64103b68f4d3b916fba52c7e650d8 is 50, key is test_row_0/A:col10/1732148858109/Put/seqid=0 2024-11-21T00:27:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:38,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742112_1288 (size=9857) 2024-11-21T00:27:38,562 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/be27d92fe7fe415abea3f2cc2f8e2182 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/be27d92fe7fe415abea3f2cc2f8e2182 2024-11-21T00:27:38,588 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into be27d92fe7fe415abea3f2cc2f8e2182(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:38,589 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:38,589 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148857997; duration=0sec 2024-11-21T00:27:38,589 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:38,589 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:38,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148918599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148918599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148918750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148918750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,863 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/b6d64103b68f4d3b916fba52c7e650d8 2024-11-21T00:27:38,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9e9d8c40e1d744c5a503fa2c0893c509 is 50, key is test_row_0/B:col10/1732148858109/Put/seqid=0 2024-11-21T00:27:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:27:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742113_1289 (size=9857) 2024-11-21T00:27:38,934 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9e9d8c40e1d744c5a503fa2c0893c509 2024-11-21T00:27:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/b0d031c1b4f24313a186d32f07ec504c is 50, key is test_row_0/C:col10/1732148858109/Put/seqid=0 2024-11-21T00:27:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148918962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148918963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:38,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742114_1290 (size=9857) 2024-11-21T00:27:38,974 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/b0d031c1b4f24313a186d32f07ec504c 2024-11-21T00:27:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/b6d64103b68f4d3b916fba52c7e650d8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b6d64103b68f4d3b916fba52c7e650d8 2024-11-21T00:27:39,012 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b6d64103b68f4d3b916fba52c7e650d8, entries=100, sequenceid=271, filesize=9.6 K 2024-11-21T00:27:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9e9d8c40e1d744c5a503fa2c0893c509 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9e9d8c40e1d744c5a503fa2c0893c509 2024-11-21T00:27:39,024 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9e9d8c40e1d744c5a503fa2c0893c509, entries=100, sequenceid=271, filesize=9.6 K 2024-11-21T00:27:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/b0d031c1b4f24313a186d32f07ec504c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b0d031c1b4f24313a186d32f07ec504c 2024-11-21T00:27:39,037 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b0d031c1b4f24313a186d32f07ec504c, entries=100, sequenceid=271, filesize=9.6 K 2024-11-21T00:27:39,038 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 9f0e4f53506bf5b3f736664c2080ec6c in 637ms, sequenceid=271, compaction requested=true 2024-11-21T00:27:39,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:39,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:39,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T00:27:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T00:27:39,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2910 sec 2024-11-21T00:27:39,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T00:27:39,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2820 sec 2024-11-21T00:27:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:39,275 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:39,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-21T00:27:39,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/c910280c85da47e4a174e4f297d6db58 is 50, key is test_row_0/A:col10/1732148859273/Put/seqid=0 2024-11-21T00:27:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148919283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148919287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742115_1291 (size=12301) 2024-11-21T00:27:39,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148919392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148919402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148919602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148919612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/c910280c85da47e4a174e4f297d6db58 2024-11-21T00:27:39,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/8ecdeee6fb744792933c773e77f38975 is 50, key is test_row_0/B:col10/1732148859273/Put/seqid=0 2024-11-21T00:27:39,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742116_1292 (size=12301) 2024-11-21T00:27:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T00:27:39,903 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:39,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T00:27:39,906 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:39,907 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:39,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:27:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148919913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:39,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148919922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:39,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:27:40,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:27:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:40,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/8ecdeee6fb744792933c773e77f38975 2024-11-21T00:27:40,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/7b9c1afaa8d04d5fb9b78583fdd57424 is 50, key is test_row_0/C:col10/1732148859273/Put/seqid=0 2024-11-21T00:27:40,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148920193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,195 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742117_1293 (size=12301) 2024-11-21T00:27:40,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/7b9c1afaa8d04d5fb9b78583fdd57424 2024-11-21T00:27:40,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148920203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,206 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:27:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:40,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:40,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:40,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148920214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,218 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:40,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/c910280c85da47e4a174e4f297d6db58 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/c910280c85da47e4a174e4f297d6db58 2024-11-21T00:27:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:27:40,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/c910280c85da47e4a174e4f297d6db58, entries=150, sequenceid=302, filesize=12.0 K 2024-11-21T00:27:40,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/8ecdeee6fb744792933c773e77f38975 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8ecdeee6fb744792933c773e77f38975 2024-11-21T00:27:40,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8ecdeee6fb744792933c773e77f38975, entries=150, sequenceid=302, filesize=12.0 K 2024-11-21T00:27:40,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/7b9c1afaa8d04d5fb9b78583fdd57424 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7b9c1afaa8d04d5fb9b78583fdd57424 2024-11-21T00:27:40,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7b9c1afaa8d04d5fb9b78583fdd57424, entries=150, sequenceid=302, filesize=12.0 K 2024-11-21T00:27:40,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 9f0e4f53506bf5b3f736664c2080ec6c in 994ms, sequenceid=302, compaction requested=true 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:40,270 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:40,270 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:40,272 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47140 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:40,272 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47140 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:40,272 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:40,272 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:40,272 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,272 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,272 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/be27d92fe7fe415abea3f2cc2f8e2182, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/dd54e5bc58e0401884121d303b664e9b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9e9d8c40e1d744c5a503fa2c0893c509, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8ecdeee6fb744792933c773e77f38975] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=46.0 K 2024-11-21T00:27:40,272 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/839b0ee9018c4a40b448351f8aeccfe6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8a4380a6b4c849318ed0cce99e7edba0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b6d64103b68f4d3b916fba52c7e650d8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/c910280c85da47e4a174e4f297d6db58] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=46.0 K 2024-11-21T00:27:40,272 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 839b0ee9018c4a40b448351f8aeccfe6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732148856643 2024-11-21T00:27:40,273 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting be27d92fe7fe415abea3f2cc2f8e2182, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732148856643 2024-11-21T00:27:40,273 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting dd54e5bc58e0401884121d303b664e9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732148857903 2024-11-21T00:27:40,273 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4380a6b4c849318ed0cce99e7edba0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732148857903 2024-11-21T00:27:40,273 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9d8c40e1d744c5a503fa2c0893c509, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148858100 2024-11-21T00:27:40,273 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d64103b68f4d3b916fba52c7e650d8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148858100 2024-11-21T00:27:40,274 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ecdeee6fb744792933c773e77f38975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732148858586 2024-11-21T00:27:40,274 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c910280c85da47e4a174e4f297d6db58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732148858586 2024-11-21T00:27:40,289 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#245 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:40,290 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7f2959776ee64f11a4dddae4b0448078 is 50, key is test_row_0/A:col10/1732148859273/Put/seqid=0 2024-11-21T00:27:40,306 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:40,307 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/545b0d16d8ab4aba936c055750c4234c is 50, key is test_row_0/B:col10/1732148859273/Put/seqid=0 2024-11-21T00:27:40,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742119_1295 (size=13017) 2024-11-21T00:27:40,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742118_1294 (size=13017) 2024-11-21T00:27:40,362 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/545b0d16d8ab4aba936c055750c4234c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/545b0d16d8ab4aba936c055750c4234c 2024-11-21T00:27:40,369 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into 545b0d16d8ab4aba936c055750c4234c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:40,369 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:40,369 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=12, startTime=1732148860270; duration=0sec 2024-11-21T00:27:40,369 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:40,369 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:40,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T00:27:40,369 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:27:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,370 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-21T00:27:40,372 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47140 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:27:40,372 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:40,372 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,372 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f0ed2f03b46e4796b855d4d774dde8d4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/fe9d9799c0954309ac1713a7de0b57aa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b0d031c1b4f24313a186d32f07ec504c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7b9c1afaa8d04d5fb9b78583fdd57424] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=46.0 K 2024-11-21T00:27:40,373 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f0ed2f03b46e4796b855d4d774dde8d4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732148856643 2024-11-21T00:27:40,373 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9d9799c0954309ac1713a7de0b57aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732148857903 2024-11-21T00:27:40,373 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d031c1b4f24313a186d32f07ec504c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148858100 2024-11-21T00:27:40,374 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b9c1afaa8d04d5fb9b78583fdd57424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732148858586 2024-11-21T00:27:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/f88ed6a7dfe44d34aa692a30aad73f31 is 50, key is test_row_0/A:col10/1732148859285/Put/seqid=0 2024-11-21T00:27:40,389 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:40,389 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/c0b0aae8c9244d21bd11f8cf7add1144 is 50, key is test_row_0/C:col10/1732148859273/Put/seqid=0 2024-11-21T00:27:40,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742120_1296 (size=7415) 2024-11-21T00:27:40,400 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/f88ed6a7dfe44d34aa692a30aad73f31 2024-11-21T00:27:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742121_1297 (size=13017) 2024-11-21T00:27:40,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/1aaa1bea7bd84398a9baf0627d60d1e3 is 50, key is test_row_0/B:col10/1732148859285/Put/seqid=0 2024-11-21T00:27:40,429 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/c0b0aae8c9244d21bd11f8cf7add1144 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0b0aae8c9244d21bd11f8cf7add1144 2024-11-21T00:27:40,439 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into c0b0aae8c9244d21bd11f8cf7add1144(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:40,439 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:40,439 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=12, startTime=1732148860270; duration=0sec 2024-11-21T00:27:40,439 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:40,439 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:40,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742122_1298 (size=7415) 2024-11-21T00:27:40,453 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/1aaa1bea7bd84398a9baf0627d60d1e3 2024-11-21T00:27:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/58b95f2855314032ade6723525983829 is 50, key is test_row_0/C:col10/1732148859285/Put/seqid=0 2024-11-21T00:27:40,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148920503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148920505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742123_1299 (size=7415) 2024-11-21T00:27:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:27:40,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148920612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148920613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,763 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7f2959776ee64f11a4dddae4b0448078 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7f2959776ee64f11a4dddae4b0448078 2024-11-21T00:27:40,772 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 7f2959776ee64f11a4dddae4b0448078(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:40,772 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:40,772 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=12, startTime=1732148860270; duration=0sec 2024-11-21T00:27:40,773 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:40,773 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:40,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148920822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148920822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:40,928 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/58b95f2855314032ade6723525983829 2024-11-21T00:27:40,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/f88ed6a7dfe44d34aa692a30aad73f31 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f88ed6a7dfe44d34aa692a30aad73f31 2024-11-21T00:27:40,939 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f88ed6a7dfe44d34aa692a30aad73f31, entries=50, sequenceid=309, filesize=7.2 K 2024-11-21T00:27:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/1aaa1bea7bd84398a9baf0627d60d1e3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/1aaa1bea7bd84398a9baf0627d60d1e3 2024-11-21T00:27:40,948 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/1aaa1bea7bd84398a9baf0627d60d1e3, entries=50, sequenceid=309, filesize=7.2 K 2024-11-21T00:27:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/58b95f2855314032ade6723525983829 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/58b95f2855314032ade6723525983829 2024-11-21T00:27:40,975 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/58b95f2855314032ade6723525983829, entries=50, sequenceid=309, filesize=7.2 K 2024-11-21T00:27:40,976 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 9f0e4f53506bf5b3f736664c2080ec6c in 607ms, sequenceid=309, compaction requested=false 2024-11-21T00:27:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T00:27:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T00:27:40,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T00:27:40,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-11-21T00:27:40,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0810 sec 2024-11-21T00:27:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T00:27:41,043 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:41,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T00:27:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:41,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:41,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:41,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:41,134 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:41,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-21T00:27:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:41,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148921152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148921154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/13396d95845a4b5ba787ec2ccff50aea is 50, key is test_row_0/A:col10/1732148861133/Put/seqid=0 2024-11-21T00:27:41,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742124_1300 (size=12301) 2024-11-21T00:27:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148921262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148921262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148921472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148921472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/13396d95845a4b5ba787ec2ccff50aea 2024-11-21T00:27:41,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/f509a6a9233847ed86eb0849bc74d7dc is 50, key is test_row_0/B:col10/1732148861133/Put/seqid=0 2024-11-21T00:27:41,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742125_1301 (size=12301) 2024-11-21T00:27:41,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/f509a6a9233847ed86eb0849bc74d7dc 2024-11-21T00:27:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:41,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:41,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:41,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/172f06bcc3da46fb890883665f4e90d4 is 50, key is test_row_0/C:col10/1732148861133/Put/seqid=0 2024-11-21T00:27:41,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742126_1302 (size=12301) 2024-11-21T00:27:41,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148921782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148921782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:41,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:41,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:42,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/172f06bcc3da46fb890883665f4e90d4 2024-11-21T00:27:42,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/13396d95845a4b5ba787ec2ccff50aea as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/13396d95845a4b5ba787ec2ccff50aea 2024-11-21T00:27:42,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/13396d95845a4b5ba787ec2ccff50aea, entries=150, sequenceid=343, filesize=12.0 K 2024-11-21T00:27:42,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/f509a6a9233847ed86eb0849bc74d7dc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f509a6a9233847ed86eb0849bc74d7dc 2024-11-21T00:27:42,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:42,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:42,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:42,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f509a6a9233847ed86eb0849bc74d7dc, entries=150, sequenceid=343, filesize=12.0 K 2024-11-21T00:27:42,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/172f06bcc3da46fb890883665f4e90d4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/172f06bcc3da46fb890883665f4e90d4 2024-11-21T00:27:42,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/172f06bcc3da46fb890883665f4e90d4, entries=150, sequenceid=343, filesize=12.0 K 2024-11-21T00:27:42,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 9f0e4f53506bf5b3f736664c2080ec6c in 1013ms, sequenceid=343, compaction requested=true 2024-11-21T00:27:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:42,148 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:42,149 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:42,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:42,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:42,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:42,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:42,150 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:42,150 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:42,150 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:42,151 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7f2959776ee64f11a4dddae4b0448078, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f88ed6a7dfe44d34aa692a30aad73f31, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/13396d95845a4b5ba787ec2ccff50aea] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=32.0 K 2024-11-21T00:27:42,151 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2959776ee64f11a4dddae4b0448078, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732148858586 2024-11-21T00:27:42,152 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88ed6a7dfe44d34aa692a30aad73f31, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148859285 2024-11-21T00:27:42,152 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:42,153 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13396d95845a4b5ba787ec2ccff50aea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732148860501 2024-11-21T00:27:42,153 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:42,153 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:42,153 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/545b0d16d8ab4aba936c055750c4234c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/1aaa1bea7bd84398a9baf0627d60d1e3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f509a6a9233847ed86eb0849bc74d7dc] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=32.0 K 2024-11-21T00:27:42,153 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 545b0d16d8ab4aba936c055750c4234c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732148858586 2024-11-21T00:27:42,154 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aaa1bea7bd84398a9baf0627d60d1e3, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148859285 2024-11-21T00:27:42,154 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f509a6a9233847ed86eb0849bc74d7dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732148860501 2024-11-21T00:27:42,178 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:42,178 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/42f760e6ba0e4820b756238b22623ba3 is 50, key is test_row_0/A:col10/1732148861133/Put/seqid=0 2024-11-21T00:27:42,181 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:42,181 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b73ee408123a4405841b753e216c13f8 is 50, key is test_row_0/B:col10/1732148861133/Put/seqid=0 2024-11-21T00:27:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:42,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742127_1303 (size=13119) 2024-11-21T00:27:42,208 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/42f760e6ba0e4820b756238b22623ba3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/42f760e6ba0e4820b756238b22623ba3 2024-11-21T00:27:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742128_1304 (size=13119) 2024-11-21T00:27:42,217 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 42f760e6ba0e4820b756238b22623ba3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:42,217 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:42,217 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148862148; duration=0sec 2024-11-21T00:27:42,218 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:42,218 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:42,218 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:42,224 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:42,224 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:42,232 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:42,232 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0b0aae8c9244d21bd11f8cf7add1144, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/58b95f2855314032ade6723525983829, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/172f06bcc3da46fb890883665f4e90d4] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=32.0 K 2024-11-21T00:27:42,233 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b0aae8c9244d21bd11f8cf7add1144, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732148858586 2024-11-21T00:27:42,235 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b73ee408123a4405841b753e216c13f8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b73ee408123a4405841b753e216c13f8 2024-11-21T00:27:42,236 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b95f2855314032ade6723525983829, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148859285 2024-11-21T00:27:42,238 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172f06bcc3da46fb890883665f4e90d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732148860501 2024-11-21T00:27:42,254 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into b73ee408123a4405841b753e216c13f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:42,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:42,254 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148862148; duration=0sec 2024-11-21T00:27:42,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:42,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:42,258 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:42,258 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/dfb2b31059354dfe8839c579353c3b82 is 50, key is test_row_0/C:col10/1732148861133/Put/seqid=0 2024-11-21T00:27:42,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T00:27:42,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:42,291 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-21T00:27:42,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742129_1305 (size=13119) 2024-11-21T00:27:42,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/4bb9202e18ef4fb5b186a70e569e0a8a is 50, key is test_row_1/A:col10/1732148861148/Put/seqid=0 2024-11-21T00:27:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:42,314 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/dfb2b31059354dfe8839c579353c3b82 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/dfb2b31059354dfe8839c579353c3b82 2024-11-21T00:27:42,328 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into dfb2b31059354dfe8839c579353c3b82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:42,328 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:42,328 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148862149; duration=0sec 2024-11-21T00:27:42,329 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:42,329 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:42,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742130_1306 (size=7415) 2024-11-21T00:27:42,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148922393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148922401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:42,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148922512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148922512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:42,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148922722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:42,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148922722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:42,740 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/4bb9202e18ef4fb5b186a70e569e0a8a 2024-11-21T00:27:42,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9a0e2bdf39de4995949ea3fc7afa4c06 is 50, key is test_row_1/B:col10/1732148861148/Put/seqid=0 2024-11-21T00:27:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742131_1307 (size=7415) 2024-11-21T00:27:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148923032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148923032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:43,206 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9a0e2bdf39de4995949ea3fc7afa4c06 2024-11-21T00:27:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/ba158e0fe6b745419a8ca2a1282a4b3f is 50, key is test_row_1/C:col10/1732148861148/Put/seqid=0 2024-11-21T00:27:43,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742132_1308 (size=7415) 2024-11-21T00:27:43,284 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/ba158e0fe6b745419a8ca2a1282a4b3f 2024-11-21T00:27:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/4bb9202e18ef4fb5b186a70e569e0a8a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/4bb9202e18ef4fb5b186a70e569e0a8a 2024-11-21T00:27:43,300 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/4bb9202e18ef4fb5b186a70e569e0a8a, entries=50, sequenceid=350, filesize=7.2 K 2024-11-21T00:27:43,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9a0e2bdf39de4995949ea3fc7afa4c06 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9a0e2bdf39de4995949ea3fc7afa4c06 2024-11-21T00:27:43,308 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9a0e2bdf39de4995949ea3fc7afa4c06, entries=50, sequenceid=350, filesize=7.2 K 2024-11-21T00:27:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/ba158e0fe6b745419a8ca2a1282a4b3f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba158e0fe6b745419a8ca2a1282a4b3f 2024-11-21T00:27:43,314 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba158e0fe6b745419a8ca2a1282a4b3f, entries=50, sequenceid=350, filesize=7.2 K 2024-11-21T00:27:43,315 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 9f0e4f53506bf5b3f736664c2080ec6c in 1024ms, sequenceid=350, compaction requested=false 2024-11-21T00:27:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T00:27:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T00:27:43,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T00:27:43,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-11-21T00:27:43,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.2750 sec 2024-11-21T00:27:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:43,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:43,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-21T00:27:43,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148923546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148923546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/cc7f81154fe240f3ab7f5f2f59d9e839 is 50, key is test_row_0/A:col10/1732148863544/Put/seqid=0 2024-11-21T00:27:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742133_1309 (size=17181) 2024-11-21T00:27:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148923662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148923662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148923872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148923872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:43,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/cc7f81154fe240f3ab7f5f2f59d9e839 2024-11-21T00:27:44,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/5a25fd30c52a493fb22c306dec87711b is 50, key is test_row_0/B:col10/1732148863544/Put/seqid=0 2024-11-21T00:27:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742134_1310 (size=12301) 2024-11-21T00:27:44,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148924182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148924184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1732148924232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,236 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45380 deadline: 1732148924244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,247 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1732148924252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,253 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 9f0e4f53506bf5b3f736664c2080ec6c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:27:44,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/5a25fd30c52a493fb22c306dec87711b 2024-11-21T00:27:44,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/2269e096b48443fe95d6a10ceca5ea80 is 50, key is test_row_0/C:col10/1732148863544/Put/seqid=0 2024-11-21T00:27:44,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742135_1311 (size=12301) 2024-11-21T00:27:44,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/2269e096b48443fe95d6a10ceca5ea80 2024-11-21T00:27:44,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/cc7f81154fe240f3ab7f5f2f59d9e839 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/cc7f81154fe240f3ab7f5f2f59d9e839 2024-11-21T00:27:44,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/cc7f81154fe240f3ab7f5f2f59d9e839, entries=250, sequenceid=384, filesize=16.8 K 2024-11-21T00:27:44,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/5a25fd30c52a493fb22c306dec87711b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/5a25fd30c52a493fb22c306dec87711b 2024-11-21T00:27:44,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/5a25fd30c52a493fb22c306dec87711b, entries=150, sequenceid=384, filesize=12.0 K 2024-11-21T00:27:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/2269e096b48443fe95d6a10ceca5ea80 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/2269e096b48443fe95d6a10ceca5ea80 2024-11-21T00:27:44,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/2269e096b48443fe95d6a10ceca5ea80, entries=150, sequenceid=384, filesize=12.0 K 2024-11-21T00:27:44,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 9f0e4f53506bf5b3f736664c2080ec6c in 1089ms, sequenceid=384, compaction requested=true 2024-11-21T00:27:44,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:44,635 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:44,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:44,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:44,636 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:44,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:44,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:44,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:44,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:44,644 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37715 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:44,644 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:44,645 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:44,645 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/42f760e6ba0e4820b756238b22623ba3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/4bb9202e18ef4fb5b186a70e569e0a8a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/cc7f81154fe240f3ab7f5f2f59d9e839] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=36.8 K 2024-11-21T00:27:44,649 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:44,649 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f760e6ba0e4820b756238b22623ba3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732148860501 2024-11-21T00:27:44,649 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:44,649 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:44,650 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b73ee408123a4405841b753e216c13f8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9a0e2bdf39de4995949ea3fc7afa4c06, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/5a25fd30c52a493fb22c306dec87711b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=32.1 K 2024-11-21T00:27:44,656 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb9202e18ef4fb5b186a70e569e0a8a, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148861148 2024-11-21T00:27:44,656 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b73ee408123a4405841b753e216c13f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732148860501 2024-11-21T00:27:44,664 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7f81154fe240f3ab7f5f2f59d9e839, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148862386 2024-11-21T00:27:44,665 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0e2bdf39de4995949ea3fc7afa4c06, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148861148 2024-11-21T00:27:44,680 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a25fd30c52a493fb22c306dec87711b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148862398 2024-11-21T00:27:44,699 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:44,700 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/2e90bb3eb088403b87769a85b7ad939d is 50, key is test_row_0/A:col10/1732148863544/Put/seqid=0 2024-11-21T00:27:44,702 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:44,703 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/ec14f52bd8e34b67b1140fe378719100 is 50, key is test_row_0/B:col10/1732148863544/Put/seqid=0 2024-11-21T00:27:44,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:44,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:44,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/bb15cf61ba6240feaff3be8ba1a63c07 is 50, key is test_row_0/A:col10/1732148864709/Put/seqid=0 2024-11-21T00:27:44,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742136_1312 (size=13221) 2024-11-21T00:27:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742137_1313 (size=13221) 2024-11-21T00:27:44,739 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/2e90bb3eb088403b87769a85b7ad939d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2e90bb3eb088403b87769a85b7ad939d 2024-11-21T00:27:44,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742138_1314 (size=12301) 2024-11-21T00:27:44,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/bb15cf61ba6240feaff3be8ba1a63c07 2024-11-21T00:27:44,746 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 2e90bb3eb088403b87769a85b7ad939d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:44,746 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:44,746 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148864635; duration=0sec 2024-11-21T00:27:44,746 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:44,747 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:44,747 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:44,748 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:44,748 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:44,748 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:44,748 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/dfb2b31059354dfe8839c579353c3b82, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba158e0fe6b745419a8ca2a1282a4b3f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/2269e096b48443fe95d6a10ceca5ea80] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=32.1 K 2024-11-21T00:27:44,749 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb2b31059354dfe8839c579353c3b82, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732148860501 2024-11-21T00:27:44,750 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ba158e0fe6b745419a8ca2a1282a4b3f, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148861148 2024-11-21T00:27:44,751 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2269e096b48443fe95d6a10ceca5ea80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148862398 2024-11-21T00:27:44,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/e5d8839de49a4956959b17856cb549aa is 50, key is test_row_0/B:col10/1732148864709/Put/seqid=0 2024-11-21T00:27:44,776 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:44,777 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/0f906a03bb334adcb2d9903fdf8d63cb is 50, key is test_row_0/C:col10/1732148863544/Put/seqid=0 2024-11-21T00:27:44,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148924792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148924803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742139_1315 (size=12301) 2024-11-21T00:27:44,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/e5d8839de49a4956959b17856cb549aa 2024-11-21T00:27:44,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/e5be784a7a3447c3a89412909a24100c is 50, key is test_row_0/C:col10/1732148864709/Put/seqid=0 2024-11-21T00:27:44,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742140_1316 (size=13221) 2024-11-21T00:27:44,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742141_1317 (size=12301) 2024-11-21T00:27:44,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/e5be784a7a3447c3a89412909a24100c 2024-11-21T00:27:44,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/bb15cf61ba6240feaff3be8ba1a63c07 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb15cf61ba6240feaff3be8ba1a63c07 2024-11-21T00:27:44,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb15cf61ba6240feaff3be8ba1a63c07, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T00:27:44,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/e5d8839de49a4956959b17856cb549aa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/e5d8839de49a4956959b17856cb549aa 2024-11-21T00:27:44,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/e5d8839de49a4956959b17856cb549aa, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T00:27:44,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/e5be784a7a3447c3a89412909a24100c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/e5be784a7a3447c3a89412909a24100c 2024-11-21T00:27:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148924912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148924916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:44,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/e5be784a7a3447c3a89412909a24100c, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T00:27:44,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9f0e4f53506bf5b3f736664c2080ec6c in 216ms, sequenceid=395, compaction requested=false 2024-11-21T00:27:44,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:45,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:45,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:27:45,139 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/ec14f52bd8e34b67b1140fe378719100 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ec14f52bd8e34b67b1140fe378719100 2024-11-21T00:27:45,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7018d09404f345e1b6ce3555a127e7b4 is 50, key is test_row_0/A:col10/1732148865124/Put/seqid=0 2024-11-21T00:27:45,163 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into ec14f52bd8e34b67b1140fe378719100(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:45,163 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,163 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148864635; duration=0sec 2024-11-21T00:27:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148925157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,163 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:45,163 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148925158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742142_1318 (size=14741) 2024-11-21T00:27:45,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7018d09404f345e1b6ce3555a127e7b4 2024-11-21T00:27:45,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/19d8c84581584907b4fa2109dca70835 is 50, key is test_row_0/B:col10/1732148865124/Put/seqid=0 2024-11-21T00:27:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T00:27:45,204 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:45,205 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T00:27:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:27:45,222 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:45,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:45,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:45,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742143_1319 (size=12301) 2024-11-21T00:27:45,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/19d8c84581584907b4fa2109dca70835 2024-11-21T00:27:45,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/ba79e52bd8194176994861b54723f3a3 is 50, key is test_row_0/C:col10/1732148865124/Put/seqid=0 2024-11-21T00:27:45,263 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/0f906a03bb334adcb2d9903fdf8d63cb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/0f906a03bb334adcb2d9903fdf8d63cb 2024-11-21T00:27:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148925272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148925276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,283 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into 0f906a03bb334adcb2d9903fdf8d63cb(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:45,283 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,284 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148864636; duration=0sec 2024-11-21T00:27:45,284 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:45,284 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:45,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742144_1320 (size=12301) 2024-11-21T00:27:45,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/ba79e52bd8194176994861b54723f3a3 2024-11-21T00:27:45,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/7018d09404f345e1b6ce3555a127e7b4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7018d09404f345e1b6ce3555a127e7b4 2024-11-21T00:27:45,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7018d09404f345e1b6ce3555a127e7b4, entries=200, sequenceid=422, filesize=14.4 K 2024-11-21T00:27:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/19d8c84581584907b4fa2109dca70835 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/19d8c84581584907b4fa2109dca70835 2024-11-21T00:27:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/19d8c84581584907b4fa2109dca70835, entries=150, sequenceid=422, filesize=12.0 K 2024-11-21T00:27:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/ba79e52bd8194176994861b54723f3a3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba79e52bd8194176994861b54723f3a3 2024-11-21T00:27:45,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba79e52bd8194176994861b54723f3a3, entries=150, sequenceid=422, filesize=12.0 K 2024-11-21T00:27:45,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9f0e4f53506bf5b3f736664c2080ec6c in 197ms, sequenceid=422, compaction requested=true 2024-11-21T00:27:45,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,323 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:45,325 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:45,325 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:45,325 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:45,325 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2e90bb3eb088403b87769a85b7ad939d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb15cf61ba6240feaff3be8ba1a63c07, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7018d09404f345e1b6ce3555a127e7b4] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=39.3 K 2024-11-21T00:27:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:45,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e90bb3eb088403b87769a85b7ad939d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148862398 2024-11-21T00:27:45,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:45,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bb15cf61ba6240feaff3be8ba1a63c07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732148864705 2024-11-21T00:27:45,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7018d09404f345e1b6ce3555a127e7b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732148864758 2024-11-21T00:27:45,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:45,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:45,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:27:45,328 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:45,329 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:45,329 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:45,329 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:45,329 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ec14f52bd8e34b67b1140fe378719100, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/e5d8839de49a4956959b17856cb549aa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/19d8c84581584907b4fa2109dca70835] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=36.9 K 2024-11-21T00:27:45,330 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec14f52bd8e34b67b1140fe378719100, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148862398 2024-11-21T00:27:45,330 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5d8839de49a4956959b17856cb549aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732148864705 2024-11-21T00:27:45,330 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d8c84581584907b4fa2109dca70835, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732148864779 2024-11-21T00:27:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:27:45,340 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:45,344 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/ce00272fd5e34e29b6643a9b74782ad8 is 50, key is test_row_0/A:col10/1732148865124/Put/seqid=0 2024-11-21T00:27:45,350 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:45,350 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/d0a98bb474a74d82812cccf1e10752d1 is 50, key is test_row_0/B:col10/1732148865124/Put/seqid=0 2024-11-21T00:27:45,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T00:27:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:45,384 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T00:27:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253 is 50, key is test_row_0/A:col10/1732148865139/Put/seqid=0 2024-11-21T00:27:45,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742145_1321 (size=13323) 2024-11-21T00:27:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742146_1322 (size=13323) 2024-11-21T00:27:45,466 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/d0a98bb474a74d82812cccf1e10752d1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d0a98bb474a74d82812cccf1e10752d1 2024-11-21T00:27:45,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742147_1323 (size=12301) 2024-11-21T00:27:45,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:27:45,544 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into d0a98bb474a74d82812cccf1e10752d1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:45,544 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,544 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148865326; duration=0sec 2024-11-21T00:27:45,544 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:45,544 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:45,544 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:45,556 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:45,556 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:45,557 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:45,557 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/0f906a03bb334adcb2d9903fdf8d63cb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/e5be784a7a3447c3a89412909a24100c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba79e52bd8194176994861b54723f3a3] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=36.9 K 2024-11-21T00:27:45,560 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f906a03bb334adcb2d9903fdf8d63cb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732148862398 2024-11-21T00:27:45,569 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5be784a7a3447c3a89412909a24100c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732148864705 2024-11-21T00:27:45,572 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba79e52bd8194176994861b54723f3a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732148864779 2024-11-21T00:27:45,642 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:45,643 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/8016b48b25f142e1a97d2f045bbb0d6d is 50, key is test_row_0/C:col10/1732148865124/Put/seqid=0 2024-11-21T00:27:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148925681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148925682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742148_1324 (size=13323) 2024-11-21T00:27:45,709 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/8016b48b25f142e1a97d2f045bbb0d6d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/8016b48b25f142e1a97d2f045bbb0d6d 2024-11-21T00:27:45,722 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into 8016b48b25f142e1a97d2f045bbb0d6d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:45,722 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,722 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148865327; duration=0sec 2024-11-21T00:27:45,722 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:45,722 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:45,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148925792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148925802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:45,826 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/ce00272fd5e34e29b6643a9b74782ad8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ce00272fd5e34e29b6643a9b74782ad8 2024-11-21T00:27:45,839 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into ce00272fd5e34e29b6643a9b74782ad8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:45,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:45,839 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148865323; duration=0sec 2024-11-21T00:27:45,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:45,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:27:45,908 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253 2024-11-21T00:27:45,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/92e847dd7dff49ad8a12d6711cea58f7 is 50, key is test_row_0/B:col10/1732148865139/Put/seqid=0 2024-11-21T00:27:45,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742149_1325 (size=12301) 2024-11-21T00:27:45,990 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/92e847dd7dff49ad8a12d6711cea58f7 2024-11-21T00:27:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f8a0e056295f434fb0372f05357e7de1 is 50, key is test_row_0/C:col10/1732148865139/Put/seqid=0 2024-11-21T00:27:46,007 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:27:46,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148926013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148926002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742150_1326 (size=12301) 2024-11-21T00:27:46,042 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f8a0e056295f434fb0372f05357e7de1 2024-11-21T00:27:46,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253 2024-11-21T00:27:46,058 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253, entries=150, sequenceid=434, filesize=12.0 K 2024-11-21T00:27:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/92e847dd7dff49ad8a12d6711cea58f7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/92e847dd7dff49ad8a12d6711cea58f7 2024-11-21T00:27:46,065 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/92e847dd7dff49ad8a12d6711cea58f7, entries=150, sequenceid=434, filesize=12.0 K 2024-11-21T00:27:46,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f8a0e056295f434fb0372f05357e7de1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f8a0e056295f434fb0372f05357e7de1 2024-11-21T00:27:46,078 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f8a0e056295f434fb0372f05357e7de1, entries=150, sequenceid=434, filesize=12.0 K 2024-11-21T00:27:46,079 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9f0e4f53506bf5b3f736664c2080ec6c in 695ms, sequenceid=434, compaction requested=false 2024-11-21T00:27:46,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:46,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T00:27:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T00:27:46,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T00:27:46,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 859 msec 2024-11-21T00:27:46,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 880 msec 2024-11-21T00:27:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:46,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:27:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:46,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/b0654baf89664509a136039d64195b5e is 50, key is test_row_0/A:col10/1732148865678/Put/seqid=0 2024-11-21T00:27:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148926346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148926350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T00:27:46,362 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:46,363 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T00:27:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:27:46,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:46,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:46,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:46,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742151_1327 (size=14741) 2024-11-21T00:27:46,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148926464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148926464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:27:46,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:27:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:27:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:27:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148926680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148926681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:46,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/b0654baf89664509a136039d64195b5e 2024-11-21T00:27:46,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9129c89cd70e409c8473197dadf7671e is 50, key is test_row_0/B:col10/1732148865678/Put/seqid=0 2024-11-21T00:27:46,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:27:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742152_1328 (size=12301) 2024-11-21T00:27:46,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9129c89cd70e409c8473197dadf7671e 2024-11-21T00:27:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:46,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/022aab94d50e4fe4a84270bb5dd6bd2c is 50, key is test_row_0/C:col10/1732148865678/Put/seqid=0 2024-11-21T00:27:46,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742153_1329 (size=12301) 2024-11-21T00:27:46,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/022aab94d50e4fe4a84270bb5dd6bd2c 2024-11-21T00:27:46,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/b0654baf89664509a136039d64195b5e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b0654baf89664509a136039d64195b5e 2024-11-21T00:27:46,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b0654baf89664509a136039d64195b5e, entries=200, sequenceid=465, filesize=14.4 K 2024-11-21T00:27:46,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/9129c89cd70e409c8473197dadf7671e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9129c89cd70e409c8473197dadf7671e 2024-11-21T00:27:46,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9129c89cd70e409c8473197dadf7671e, entries=150, sequenceid=465, filesize=12.0 K 2024-11-21T00:27:46,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/022aab94d50e4fe4a84270bb5dd6bd2c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/022aab94d50e4fe4a84270bb5dd6bd2c 2024-11-21T00:27:46,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/022aab94d50e4fe4a84270bb5dd6bd2c, entries=150, sequenceid=465, filesize=12.0 K 2024-11-21T00:27:46,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9f0e4f53506bf5b3f736664c2080ec6c in 614ms, sequenceid=465, compaction requested=true 2024-11-21T00:27:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:46,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:46,950 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:46,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:46,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:46,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:46,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:46,950 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:46,950 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:46,950 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,950 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ce00272fd5e34e29b6643a9b74782ad8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b0654baf89664509a136039d64195b5e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=39.4 K 2024-11-21T00:27:46,951 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce00272fd5e34e29b6643a9b74782ad8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732148864779 2024-11-21T00:27:46,951 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:46,951 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:46,951 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:46,951 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d0a98bb474a74d82812cccf1e10752d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/92e847dd7dff49ad8a12d6711cea58f7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9129c89cd70e409c8473197dadf7671e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=37.0 K 2024-11-21T00:27:46,951 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa8ad087f9f4ea7b2a9f2cc07bf8253, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732148865127 2024-11-21T00:27:46,951 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a98bb474a74d82812cccf1e10752d1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732148864779 2024-11-21T00:27:46,956 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0654baf89664509a136039d64195b5e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732148865675 2024-11-21T00:27:46,958 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e847dd7dff49ad8a12d6711cea58f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732148865127 2024-11-21T00:27:46,959 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9129c89cd70e409c8473197dadf7671e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732148865678 2024-11-21T00:27:46,981 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:46,981 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/a234bc1e08f7493d84a6fdf612c9f204 is 50, key is test_row_0/A:col10/1732148865678/Put/seqid=0 2024-11-21T00:27:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:27:46,997 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:46,998 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b4c9b3ed45e64125adf032637de58182 is 50, key is test_row_0/B:col10/1732148865678/Put/seqid=0 2024-11-21T00:27:47,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T00:27:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:47,001 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T00:27:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:47,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. as already flushing 2024-11-21T00:27:47,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742154_1330 (size=13425) 2024-11-21T00:27:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/732e5cc0df20451aa0e4dd7192e3c794 is 50, key is test_row_0/A:col10/1732148866995/Put/seqid=0 2024-11-21T00:27:47,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742155_1331 (size=13425) 2024-11-21T00:27:47,049 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/b4c9b3ed45e64125adf032637de58182 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b4c9b3ed45e64125adf032637de58182 2024-11-21T00:27:47,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742156_1332 (size=12301) 2024-11-21T00:27:47,055 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into b4c9b3ed45e64125adf032637de58182(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:47,055 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:47,055 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148866949; duration=0sec 2024-11-21T00:27:47,055 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:47,055 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:47,055 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:47,057 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:47,057 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:47,057 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:47,057 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/8016b48b25f142e1a97d2f045bbb0d6d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f8a0e056295f434fb0372f05357e7de1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/022aab94d50e4fe4a84270bb5dd6bd2c] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=37.0 K 2024-11-21T00:27:47,057 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8016b48b25f142e1a97d2f045bbb0d6d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732148864779 2024-11-21T00:27:47,058 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a0e056295f434fb0372f05357e7de1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732148865127 2024-11-21T00:27:47,058 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 022aab94d50e4fe4a84270bb5dd6bd2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732148865678 2024-11-21T00:27:47,081 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:47,082 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/1800fe1ae27b4539982d02157e6eba04 is 50, key is test_row_0/C:col10/1732148865678/Put/seqid=0 2024-11-21T00:27:47,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742157_1333 (size=13425) 2024-11-21T00:27:47,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148927117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148927119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,129 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/1800fe1ae27b4539982d02157e6eba04 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/1800fe1ae27b4539982d02157e6eba04 2024-11-21T00:27:47,148 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into 1800fe1ae27b4539982d02157e6eba04(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:47,148 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:47,148 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148866950; duration=0sec 2024-11-21T00:27:47,149 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:47,149 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148927232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148927232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,269 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-21T00:27:47,269 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:47,269 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:47,270 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-21T00:27:47,270 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:47,270 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,270 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,270 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-11-21T00:27:47,270 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:47,270 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,270 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,270 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:47,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:47,273 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-21T00:27:47,273 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:47,273 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,273 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:47,274 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-21T00:27:47,274 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:47,274 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,274 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:47,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:47,442 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/a234bc1e08f7493d84a6fdf612c9f204 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/a234bc1e08f7493d84a6fdf612c9f204 2024-11-21T00:27:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148927442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148927444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,449 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into a234bc1e08f7493d84a6fdf612c9f204(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:47,449 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:47,449 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148866949; duration=0sec 2024-11-21T00:27:47,449 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:47,449 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:47,456 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/732e5cc0df20451aa0e4dd7192e3c794 2024-11-21T00:27:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/98dee8eaea4e4337a52b857012fee1e8 is 50, key is test_row_0/B:col10/1732148866995/Put/seqid=0 2024-11-21T00:27:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:27:47,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742158_1334 (size=12301) 2024-11-21T00:27:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/98dee8eaea4e4337a52b857012fee1e8 2024-11-21T00:27:47,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/728edee8974b40a8877421713f379a7e is 50, key is test_row_0/C:col10/1732148866995/Put/seqid=0 2024-11-21T00:27:47,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742159_1335 (size=12301) 2024-11-21T00:27:47,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45376 deadline: 1732148927753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45360 deadline: 1732148927756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:27:47,990 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/728edee8974b40a8877421713f379a7e 2024-11-21T00:27:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/732e5cc0df20451aa0e4dd7192e3c794 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/732e5cc0df20451aa0e4dd7192e3c794 2024-11-21T00:27:48,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/732e5cc0df20451aa0e4dd7192e3c794, entries=150, sequenceid=474, filesize=12.0 K 2024-11-21T00:27:48,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/98dee8eaea4e4337a52b857012fee1e8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/98dee8eaea4e4337a52b857012fee1e8 2024-11-21T00:27:48,017 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/98dee8eaea4e4337a52b857012fee1e8, entries=150, sequenceid=474, filesize=12.0 K 2024-11-21T00:27:48,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/728edee8974b40a8877421713f379a7e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/728edee8974b40a8877421713f379a7e 2024-11-21T00:27:48,024 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/728edee8974b40a8877421713f379a7e, entries=150, sequenceid=474, filesize=12.0 K 2024-11-21T00:27:48,026 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 9f0e4f53506bf5b3f736664c2080ec6c in 1025ms, sequenceid=474, compaction requested=false 2024-11-21T00:27:48,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:48,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:48,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T00:27:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T00:27:48,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T00:27:48,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-21T00:27:48,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6690 sec 2024-11-21T00:27:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:48,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T00:27:48,263 INFO [Thread-1080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1080. 2024-11-21T00:27:48,263 DEBUG [Thread-1080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:48,263 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:48,263 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:48,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:48,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/ebabdff97e554c39bd0ece1879695b44 is 50, key is test_row_0/A:col10/1732148868262/Put/seqid=0 2024-11-21T00:27:48,273 INFO [Thread-1078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1078. 2024-11-21T00:27:48,273 DEBUG [Thread-1078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:48,273 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:48,274 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:48,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:48,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742160_1336 (size=12301) 2024-11-21T00:27:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T00:27:48,513 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:48,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/ebabdff97e554c39bd0ece1879695b44 2024-11-21T00:27:48,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/888393fb6e2b4137874a810ded3b3f78 is 50, key is test_row_0/B:col10/1732148868262/Put/seqid=0 2024-11-21T00:27:48,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742161_1337 (size=12301) 2024-11-21T00:27:49,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/888393fb6e2b4137874a810ded3b3f78 2024-11-21T00:27:49,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f1baa4ca2b7d42f0a2fff42728318fe2 is 50, key is test_row_0/C:col10/1732148868262/Put/seqid=0 2024-11-21T00:27:49,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742162_1338 (size=12301) 2024-11-21T00:27:49,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f1baa4ca2b7d42f0a2fff42728318fe2 2024-11-21T00:27:49,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/ebabdff97e554c39bd0ece1879695b44 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ebabdff97e554c39bd0ece1879695b44 2024-11-21T00:27:49,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ebabdff97e554c39bd0ece1879695b44, entries=150, sequenceid=505, filesize=12.0 K 2024-11-21T00:27:49,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/888393fb6e2b4137874a810ded3b3f78 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/888393fb6e2b4137874a810ded3b3f78 2024-11-21T00:27:49,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/888393fb6e2b4137874a810ded3b3f78, entries=150, sequenceid=505, filesize=12.0 K 2024-11-21T00:27:49,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/f1baa4ca2b7d42f0a2fff42728318fe2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f1baa4ca2b7d42f0a2fff42728318fe2 2024-11-21T00:27:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f1baa4ca2b7d42f0a2fff42728318fe2, entries=150, sequenceid=505, filesize=12.0 K 2024-11-21T00:27:49,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=6.71 KB/6870 for 9f0e4f53506bf5b3f736664c2080ec6c in 1305ms, sequenceid=505, compaction requested=true 2024-11-21T00:27:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:49,569 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:49,569 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f0e4f53506bf5b3f736664c2080ec6c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:49,570 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:49,570 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/A is initiating minor compaction (all files) 2024-11-21T00:27:49,570 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/A in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:49,570 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/a234bc1e08f7493d84a6fdf612c9f204, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/732e5cc0df20451aa0e4dd7192e3c794, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ebabdff97e554c39bd0ece1879695b44] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=37.1 K 2024-11-21T00:27:49,570 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:49,570 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/B is initiating minor compaction (all files) 2024-11-21T00:27:49,570 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/B in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:49,570 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b4c9b3ed45e64125adf032637de58182, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/98dee8eaea4e4337a52b857012fee1e8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/888393fb6e2b4137874a810ded3b3f78] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=37.1 K 2024-11-21T00:27:49,572 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c9b3ed45e64125adf032637de58182, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732148865678 2024-11-21T00:27:49,572 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a234bc1e08f7493d84a6fdf612c9f204, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732148865678 2024-11-21T00:27:49,573 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 98dee8eaea4e4337a52b857012fee1e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732148866339 2024-11-21T00:27:49,573 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732e5cc0df20451aa0e4dd7192e3c794, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732148866339 2024-11-21T00:27:49,573 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 888393fb6e2b4137874a810ded3b3f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1732148867098 2024-11-21T00:27:49,573 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebabdff97e554c39bd0ece1879695b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1732148867098 2024-11-21T00:27:49,594 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:49,594 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/8aec8843954f4e1e81f6c9c38ff7223f is 50, key is test_row_0/B:col10/1732148868262/Put/seqid=0 2024-11-21T00:27:49,601 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#A#compaction#291 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:49,602 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/375f7725dfe94f72a7511600ff6a0c48 is 50, key is test_row_0/A:col10/1732148868262/Put/seqid=0 2024-11-21T00:27:49,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742163_1339 (size=13527) 2024-11-21T00:27:49,627 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/8aec8843954f4e1e81f6c9c38ff7223f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8aec8843954f4e1e81f6c9c38ff7223f 2024-11-21T00:27:49,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742164_1340 (size=13527) 2024-11-21T00:27:49,633 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/B of 9f0e4f53506bf5b3f736664c2080ec6c into 8aec8843954f4e1e81f6c9c38ff7223f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:49,633 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:49,633 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/B, priority=13, startTime=1732148869569; duration=0sec 2024-11-21T00:27:49,633 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:49,633 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:B 2024-11-21T00:27:49,633 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:49,638 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:49,638 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 9f0e4f53506bf5b3f736664c2080ec6c/C is initiating minor compaction (all files) 2024-11-21T00:27:49,638 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f0e4f53506bf5b3f736664c2080ec6c/C in TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:49,638 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/1800fe1ae27b4539982d02157e6eba04, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/728edee8974b40a8877421713f379a7e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f1baa4ca2b7d42f0a2fff42728318fe2] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp, totalSize=37.1 K 2024-11-21T00:27:49,639 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/375f7725dfe94f72a7511600ff6a0c48 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/375f7725dfe94f72a7511600ff6a0c48 2024-11-21T00:27:49,640 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1800fe1ae27b4539982d02157e6eba04, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732148865678 2024-11-21T00:27:49,640 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 728edee8974b40a8877421713f379a7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732148866339 2024-11-21T00:27:49,641 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f1baa4ca2b7d42f0a2fff42728318fe2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1732148867098 2024-11-21T00:27:49,646 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/A of 9f0e4f53506bf5b3f736664c2080ec6c into 375f7725dfe94f72a7511600ff6a0c48(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:49,646 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:49,646 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/A, priority=13, startTime=1732148869569; duration=0sec 2024-11-21T00:27:49,646 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:49,646 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:A 2024-11-21T00:27:49,652 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f0e4f53506bf5b3f736664c2080ec6c#C#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:49,653 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/9d2e92079b7b4a49ad23a373217b574a is 50, key is test_row_0/C:col10/1732148868262/Put/seqid=0 2024-11-21T00:27:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742165_1341 (size=13527) 2024-11-21T00:27:49,671 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/9d2e92079b7b4a49ad23a373217b574a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9d2e92079b7b4a49ad23a373217b574a 2024-11-21T00:27:49,676 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f0e4f53506bf5b3f736664c2080ec6c/C of 9f0e4f53506bf5b3f736664c2080ec6c into 9d2e92079b7b4a49ad23a373217b574a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:49,676 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f0e4f53506bf5b3f736664c2080ec6c: 2024-11-21T00:27:49,676 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., storeName=9f0e4f53506bf5b3f736664c2080ec6c/C, priority=13, startTime=1732148869569; duration=0sec 2024-11-21T00:27:49,676 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:49,676 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f0e4f53506bf5b3f736664c2080ec6c:C 2024-11-21T00:27:54,277 INFO [Thread-1082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1082. 2024-11-21T00:27:54,277 DEBUG [Thread-1082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:54,277 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,277 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:54,309 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-21T00:27:54,309 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:54,309 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,309 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:54,334 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-11-21T00:27:54,334 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:27:54,334 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,334 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:54,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:27:54,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-21T00:27:54,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2800 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2756 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2865 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2788 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2699 2024-11-21T00:27:54,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:27:54,335 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:27:54,335 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:27:54,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:54,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:27:54,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:27:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:27:54,339 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148874339"}]},"ts":"1732148874339"} 2024-11-21T00:27:54,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:27:54,340 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:27:54,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:27:54,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, UNASSIGN}] 2024-11-21T00:27:54,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, UNASSIGN 2024-11-21T00:27:54,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=9f0e4f53506bf5b3f736664c2080ec6c, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:54,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, UNASSIGN because future has completed 2024-11-21T00:27:54,346 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:27:54,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:27:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:27:54,498 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:54,498 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:27:54,498 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 9f0e4f53506bf5b3f736664c2080ec6c, disabling compactions & flushes 2024-11-21T00:27:54,498 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:54,499 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:54,499 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. after waiting 0 ms 2024-11-21T00:27:54,499 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:54,499 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 9f0e4f53506bf5b3f736664c2080ec6c 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T00:27:54,506 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/1efd67af34dc4c9f862f46c3a8c2d235 is 50, key is test_row_0/A:col10/1732148874302/Put/seqid=0 2024-11-21T00:27:54,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742166_1342 (size=12301) 2024-11-21T00:27:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:27:54,915 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/1efd67af34dc4c9f862f46c3a8c2d235 2024-11-21T00:27:54,922 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/0da9ec6355ac4659ad803ea66ce5061b is 50, key is test_row_0/B:col10/1732148874302/Put/seqid=0 2024-11-21T00:27:54,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742167_1343 (size=12301) 2024-11-21T00:27:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:27:55,326 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/0da9ec6355ac4659ad803ea66ce5061b 2024-11-21T00:27:55,332 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/18bc2edfaa774a87ac66b6da9dc9697e is 50, key is test_row_0/C:col10/1732148874302/Put/seqid=0 2024-11-21T00:27:55,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742168_1344 (size=12301) 2024-11-21T00:27:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:27:55,735 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/18bc2edfaa774a87ac66b6da9dc9697e 2024-11-21T00:27:55,739 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/A/1efd67af34dc4c9f862f46c3a8c2d235 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/1efd67af34dc4c9f862f46c3a8c2d235 2024-11-21T00:27:55,743 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/1efd67af34dc4c9f862f46c3a8c2d235, entries=150, sequenceid=515, filesize=12.0 K 2024-11-21T00:27:55,744 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/B/0da9ec6355ac4659ad803ea66ce5061b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0da9ec6355ac4659ad803ea66ce5061b 2024-11-21T00:27:55,748 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0da9ec6355ac4659ad803ea66ce5061b, entries=150, sequenceid=515, filesize=12.0 K 2024-11-21T00:27:55,748 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/.tmp/C/18bc2edfaa774a87ac66b6da9dc9697e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/18bc2edfaa774a87ac66b6da9dc9697e 2024-11-21T00:27:55,752 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/18bc2edfaa774a87ac66b6da9dc9697e, entries=150, sequenceid=515, filesize=12.0 K 2024-11-21T00:27:55,753 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9f0e4f53506bf5b3f736664c2080ec6c in 1254ms, sequenceid=515, compaction requested=false 2024-11-21T00:27:55,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/6103607deb38485c874e3b60841490e6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/eb7a13a0570541da9713f02631c2d462, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7d1b4bf0d5bb49adb178751a287c79b6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/5657256458e24d1890b6eb22cd5b1c7a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/17c83b60c0f3450f8c5ccb9917cd6759, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f09a96897e274e6597ac8b83e80b74d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2a7ec4588b5143acb09a71cff5f68eaf, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3d99332134cc41b7858779a41c12f2a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9492a85facbf457f8149e30ff698f6ab, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/887c7b3dad9c4ed1a21f1d6507f9354a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb2f8378546e40c983fceff830965917, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/d3f2f50decae43f0a77f6c1b2db23c8a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9cbe6c6c119943608cf9068644b112ef, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8268e028b0f34839b3f148ed65253a5b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7a07fca49d2a42feb789ed179032517e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/e8300da868d54c4ba8e6215af0ffcb26, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/58060e1ef1b14371bb654d0c49ea562f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/839b0ee9018c4a40b448351f8aeccfe6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8a4380a6b4c849318ed0cce99e7edba0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b6d64103b68f4d3b916fba52c7e650d8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7f2959776ee64f11a4dddae4b0448078, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/c910280c85da47e4a174e4f297d6db58, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f88ed6a7dfe44d34aa692a30aad73f31, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/42f760e6ba0e4820b756238b22623ba3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/13396d95845a4b5ba787ec2ccff50aea, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/4bb9202e18ef4fb5b186a70e569e0a8a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/cc7f81154fe240f3ab7f5f2f59d9e839, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2e90bb3eb088403b87769a85b7ad939d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb15cf61ba6240feaff3be8ba1a63c07, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7018d09404f345e1b6ce3555a127e7b4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ce00272fd5e34e29b6643a9b74782ad8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b0654baf89664509a136039d64195b5e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/a234bc1e08f7493d84a6fdf612c9f204, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/732e5cc0df20451aa0e4dd7192e3c794, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ebabdff97e554c39bd0ece1879695b44] to archive 2024-11-21T00:27:55,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:27:55,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/6103607deb38485c874e3b60841490e6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/6103607deb38485c874e3b60841490e6 2024-11-21T00:27:55,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/eb7a13a0570541da9713f02631c2d462 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/eb7a13a0570541da9713f02631c2d462 2024-11-21T00:27:55,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7d1b4bf0d5bb49adb178751a287c79b6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7d1b4bf0d5bb49adb178751a287c79b6 2024-11-21T00:27:55,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/5657256458e24d1890b6eb22cd5b1c7a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/5657256458e24d1890b6eb22cd5b1c7a 2024-11-21T00:27:55,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/17c83b60c0f3450f8c5ccb9917cd6759 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/17c83b60c0f3450f8c5ccb9917cd6759 2024-11-21T00:27:55,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f09a96897e274e6597ac8b83e80b74d5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f09a96897e274e6597ac8b83e80b74d5 2024-11-21T00:27:55,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2a7ec4588b5143acb09a71cff5f68eaf to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2a7ec4588b5143acb09a71cff5f68eaf 2024-11-21T00:27:55,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3d99332134cc41b7858779a41c12f2a6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3d99332134cc41b7858779a41c12f2a6 2024-11-21T00:27:55,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9492a85facbf457f8149e30ff698f6ab to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9492a85facbf457f8149e30ff698f6ab 2024-11-21T00:27:55,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/887c7b3dad9c4ed1a21f1d6507f9354a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/887c7b3dad9c4ed1a21f1d6507f9354a 2024-11-21T00:27:55,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb2f8378546e40c983fceff830965917 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb2f8378546e40c983fceff830965917 2024-11-21T00:27:55,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/d3f2f50decae43f0a77f6c1b2db23c8a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/d3f2f50decae43f0a77f6c1b2db23c8a 2024-11-21T00:27:55,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9cbe6c6c119943608cf9068644b112ef to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/9cbe6c6c119943608cf9068644b112ef 2024-11-21T00:27:55,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8268e028b0f34839b3f148ed65253a5b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8268e028b0f34839b3f148ed65253a5b 2024-11-21T00:27:55,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7a07fca49d2a42feb789ed179032517e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7a07fca49d2a42feb789ed179032517e 2024-11-21T00:27:55,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/e8300da868d54c4ba8e6215af0ffcb26 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/e8300da868d54c4ba8e6215af0ffcb26 2024-11-21T00:27:55,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/58060e1ef1b14371bb654d0c49ea562f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/58060e1ef1b14371bb654d0c49ea562f 2024-11-21T00:27:55,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/839b0ee9018c4a40b448351f8aeccfe6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/839b0ee9018c4a40b448351f8aeccfe6 2024-11-21T00:27:55,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8a4380a6b4c849318ed0cce99e7edba0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/8a4380a6b4c849318ed0cce99e7edba0 2024-11-21T00:27:55,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b6d64103b68f4d3b916fba52c7e650d8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b6d64103b68f4d3b916fba52c7e650d8 2024-11-21T00:27:55,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7f2959776ee64f11a4dddae4b0448078 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7f2959776ee64f11a4dddae4b0448078 2024-11-21T00:27:55,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/c910280c85da47e4a174e4f297d6db58 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/c910280c85da47e4a174e4f297d6db58 2024-11-21T00:27:55,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f88ed6a7dfe44d34aa692a30aad73f31 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/f88ed6a7dfe44d34aa692a30aad73f31 2024-11-21T00:27:55,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/42f760e6ba0e4820b756238b22623ba3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/42f760e6ba0e4820b756238b22623ba3 2024-11-21T00:27:55,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/13396d95845a4b5ba787ec2ccff50aea to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/13396d95845a4b5ba787ec2ccff50aea 2024-11-21T00:27:55,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/4bb9202e18ef4fb5b186a70e569e0a8a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/4bb9202e18ef4fb5b186a70e569e0a8a 2024-11-21T00:27:55,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/cc7f81154fe240f3ab7f5f2f59d9e839 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/cc7f81154fe240f3ab7f5f2f59d9e839 2024-11-21T00:27:55,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2e90bb3eb088403b87769a85b7ad939d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/2e90bb3eb088403b87769a85b7ad939d 2024-11-21T00:27:55,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb15cf61ba6240feaff3be8ba1a63c07 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/bb15cf61ba6240feaff3be8ba1a63c07 2024-11-21T00:27:55,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7018d09404f345e1b6ce3555a127e7b4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/7018d09404f345e1b6ce3555a127e7b4 2024-11-21T00:27:55,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ce00272fd5e34e29b6643a9b74782ad8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ce00272fd5e34e29b6643a9b74782ad8 2024-11-21T00:27:55,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/3aa8ad087f9f4ea7b2a9f2cc07bf8253 2024-11-21T00:27:55,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b0654baf89664509a136039d64195b5e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/b0654baf89664509a136039d64195b5e 2024-11-21T00:27:55,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/a234bc1e08f7493d84a6fdf612c9f204 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/a234bc1e08f7493d84a6fdf612c9f204 2024-11-21T00:27:55,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/732e5cc0df20451aa0e4dd7192e3c794 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/732e5cc0df20451aa0e4dd7192e3c794 2024-11-21T00:27:55,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ebabdff97e554c39bd0ece1879695b44 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/ebabdff97e554c39bd0ece1879695b44 2024-11-21T00:27:55,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b6bfa256e4a840f49a88f3447b8c2237, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/74bd3703b36b444a86de5f0fa618d19d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/fe14bcd449d9468985d73bc508b61967, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ee5a060025484e0d816cb23a2a4ebc65, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/432c3677ba7d480b98c695730a6938dd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/4605189d023a40e2b32e8747b2788a91, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a4d2d65df3fa4082ab875b184315af3c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d83489c4371149d281d88f91f0966e93, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/40b244f38af34b8f9ac03f16b59d94c9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0f872ac07ee940f5ad1bde9efcf17ad3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/6e56360c6c5d408788a3824de45db512, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/3298037bc5de404882f0709825ec5c4e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/7bcb90468cf74d009cc53a989cb4ad01, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a68db8825c4c4391a202df398cab44f5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f570c8fd1f124a0ea6ebd04452252f02, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9fb0c3701baf468da6cd1616b40d92d3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/be27d92fe7fe415abea3f2cc2f8e2182, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/bda0e6f5fb7d4a068a2e5065e66f1608, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/dd54e5bc58e0401884121d303b664e9b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9e9d8c40e1d744c5a503fa2c0893c509, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/545b0d16d8ab4aba936c055750c4234c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8ecdeee6fb744792933c773e77f38975, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/1aaa1bea7bd84398a9baf0627d60d1e3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b73ee408123a4405841b753e216c13f8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f509a6a9233847ed86eb0849bc74d7dc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9a0e2bdf39de4995949ea3fc7afa4c06, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ec14f52bd8e34b67b1140fe378719100, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/5a25fd30c52a493fb22c306dec87711b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/e5d8839de49a4956959b17856cb549aa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d0a98bb474a74d82812cccf1e10752d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/19d8c84581584907b4fa2109dca70835, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/92e847dd7dff49ad8a12d6711cea58f7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b4c9b3ed45e64125adf032637de58182, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9129c89cd70e409c8473197dadf7671e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/98dee8eaea4e4337a52b857012fee1e8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/888393fb6e2b4137874a810ded3b3f78] to archive 2024-11-21T00:27:55,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:27:55,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b6bfa256e4a840f49a88f3447b8c2237 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b6bfa256e4a840f49a88f3447b8c2237 2024-11-21T00:27:55,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/74bd3703b36b444a86de5f0fa618d19d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/74bd3703b36b444a86de5f0fa618d19d 2024-11-21T00:27:55,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/fe14bcd449d9468985d73bc508b61967 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/fe14bcd449d9468985d73bc508b61967 2024-11-21T00:27:55,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ee5a060025484e0d816cb23a2a4ebc65 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ee5a060025484e0d816cb23a2a4ebc65 2024-11-21T00:27:55,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/432c3677ba7d480b98c695730a6938dd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/432c3677ba7d480b98c695730a6938dd 2024-11-21T00:27:55,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/4605189d023a40e2b32e8747b2788a91 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/4605189d023a40e2b32e8747b2788a91 2024-11-21T00:27:55,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a4d2d65df3fa4082ab875b184315af3c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a4d2d65df3fa4082ab875b184315af3c 2024-11-21T00:27:55,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d83489c4371149d281d88f91f0966e93 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d83489c4371149d281d88f91f0966e93 2024-11-21T00:27:55,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/40b244f38af34b8f9ac03f16b59d94c9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/40b244f38af34b8f9ac03f16b59d94c9 2024-11-21T00:27:55,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0f872ac07ee940f5ad1bde9efcf17ad3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0f872ac07ee940f5ad1bde9efcf17ad3 2024-11-21T00:27:55,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/6e56360c6c5d408788a3824de45db512 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/6e56360c6c5d408788a3824de45db512 2024-11-21T00:27:55,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/3298037bc5de404882f0709825ec5c4e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/3298037bc5de404882f0709825ec5c4e 2024-11-21T00:27:55,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/7bcb90468cf74d009cc53a989cb4ad01 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/7bcb90468cf74d009cc53a989cb4ad01 2024-11-21T00:27:55,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a68db8825c4c4391a202df398cab44f5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/a68db8825c4c4391a202df398cab44f5 2024-11-21T00:27:55,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f570c8fd1f124a0ea6ebd04452252f02 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f570c8fd1f124a0ea6ebd04452252f02 2024-11-21T00:27:55,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9fb0c3701baf468da6cd1616b40d92d3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9fb0c3701baf468da6cd1616b40d92d3 2024-11-21T00:27:55,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/be27d92fe7fe415abea3f2cc2f8e2182 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/be27d92fe7fe415abea3f2cc2f8e2182 2024-11-21T00:27:55,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/bda0e6f5fb7d4a068a2e5065e66f1608 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/bda0e6f5fb7d4a068a2e5065e66f1608 2024-11-21T00:27:55,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/dd54e5bc58e0401884121d303b664e9b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/dd54e5bc58e0401884121d303b664e9b 2024-11-21T00:27:55,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9e9d8c40e1d744c5a503fa2c0893c509 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9e9d8c40e1d744c5a503fa2c0893c509 2024-11-21T00:27:55,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/545b0d16d8ab4aba936c055750c4234c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/545b0d16d8ab4aba936c055750c4234c 2024-11-21T00:27:55,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8ecdeee6fb744792933c773e77f38975 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8ecdeee6fb744792933c773e77f38975 2024-11-21T00:27:55,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/1aaa1bea7bd84398a9baf0627d60d1e3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/1aaa1bea7bd84398a9baf0627d60d1e3 2024-11-21T00:27:55,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b73ee408123a4405841b753e216c13f8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b73ee408123a4405841b753e216c13f8 2024-11-21T00:27:55,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f509a6a9233847ed86eb0849bc74d7dc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/f509a6a9233847ed86eb0849bc74d7dc 2024-11-21T00:27:55,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9a0e2bdf39de4995949ea3fc7afa4c06 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9a0e2bdf39de4995949ea3fc7afa4c06 2024-11-21T00:27:55,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ec14f52bd8e34b67b1140fe378719100 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/ec14f52bd8e34b67b1140fe378719100 2024-11-21T00:27:55,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/5a25fd30c52a493fb22c306dec87711b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/5a25fd30c52a493fb22c306dec87711b 2024-11-21T00:27:55,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/e5d8839de49a4956959b17856cb549aa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/e5d8839de49a4956959b17856cb549aa 2024-11-21T00:27:55,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d0a98bb474a74d82812cccf1e10752d1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/d0a98bb474a74d82812cccf1e10752d1 2024-11-21T00:27:55,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/19d8c84581584907b4fa2109dca70835 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/19d8c84581584907b4fa2109dca70835 2024-11-21T00:27:55,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/92e847dd7dff49ad8a12d6711cea58f7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/92e847dd7dff49ad8a12d6711cea58f7 2024-11-21T00:27:55,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b4c9b3ed45e64125adf032637de58182 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/b4c9b3ed45e64125adf032637de58182 2024-11-21T00:27:55,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9129c89cd70e409c8473197dadf7671e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/9129c89cd70e409c8473197dadf7671e 2024-11-21T00:27:55,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/98dee8eaea4e4337a52b857012fee1e8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/98dee8eaea4e4337a52b857012fee1e8 2024-11-21T00:27:55,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/888393fb6e2b4137874a810ded3b3f78 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/888393fb6e2b4137874a810ded3b3f78 2024-11-21T00:27:55,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15f9c3bd0905490db107bee7c489c280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/4082fee3919b447a91c81a05f3fa1439, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/27dcc589326741bcb1aa35b904b928d5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/60ac478751b0495b9e00e12d1171325c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15b6f0a3cf2f4a5895cf07cf353f0784, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/763d1daa6ea84b7c8278f7566c0d813d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cfc39461ad82473e878d287c9571c107, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/43a2280b481a4f3c99986a737be46757, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cdc5ee13f0b64ccf978866f5300df3dd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0d647b1acdf4cbea6dbd86645d27e59, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/12e1b94eced3456ab43407ad3d372689, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a7af240c11524c96ad00db0089a1b74f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7f928af45184487cb36cd91256a5b1d9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9022793c0fe445b4b80d829265ef0d91, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/af4efc562bb04d80bd114f33198b87b9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b67d266f09e24bd0ad8d03e056c8ebcb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f0ed2f03b46e4796b855d4d774dde8d4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a4097c4bed4e4c2b94929e0511e49586, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/fe9d9799c0954309ac1713a7de0b57aa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b0d031c1b4f24313a186d32f07ec504c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0b0aae8c9244d21bd11f8cf7add1144, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7b9c1afaa8d04d5fb9b78583fdd57424, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/58b95f2855314032ade6723525983829, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/dfb2b31059354dfe8839c579353c3b82, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/172f06bcc3da46fb890883665f4e90d4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba158e0fe6b745419a8ca2a1282a4b3f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/0f906a03bb334adcb2d9903fdf8d63cb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/2269e096b48443fe95d6a10ceca5ea80, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/e5be784a7a3447c3a89412909a24100c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/8016b48b25f142e1a97d2f045bbb0d6d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba79e52bd8194176994861b54723f3a3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f8a0e056295f434fb0372f05357e7de1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/1800fe1ae27b4539982d02157e6eba04, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/022aab94d50e4fe4a84270bb5dd6bd2c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/728edee8974b40a8877421713f379a7e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f1baa4ca2b7d42f0a2fff42728318fe2] to archive 2024-11-21T00:27:55,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:27:55,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15f9c3bd0905490db107bee7c489c280 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15f9c3bd0905490db107bee7c489c280 2024-11-21T00:27:55,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/4082fee3919b447a91c81a05f3fa1439 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/4082fee3919b447a91c81a05f3fa1439 2024-11-21T00:27:55,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/27dcc589326741bcb1aa35b904b928d5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/27dcc589326741bcb1aa35b904b928d5 2024-11-21T00:27:55,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/60ac478751b0495b9e00e12d1171325c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/60ac478751b0495b9e00e12d1171325c 2024-11-21T00:27:55,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15b6f0a3cf2f4a5895cf07cf353f0784 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/15b6f0a3cf2f4a5895cf07cf353f0784 2024-11-21T00:27:55,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/763d1daa6ea84b7c8278f7566c0d813d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/763d1daa6ea84b7c8278f7566c0d813d 2024-11-21T00:27:55,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cfc39461ad82473e878d287c9571c107 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cfc39461ad82473e878d287c9571c107 2024-11-21T00:27:55,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/43a2280b481a4f3c99986a737be46757 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/43a2280b481a4f3c99986a737be46757 2024-11-21T00:27:55,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cdc5ee13f0b64ccf978866f5300df3dd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/cdc5ee13f0b64ccf978866f5300df3dd 2024-11-21T00:27:55,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0d647b1acdf4cbea6dbd86645d27e59 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0d647b1acdf4cbea6dbd86645d27e59 2024-11-21T00:27:55,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/12e1b94eced3456ab43407ad3d372689 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/12e1b94eced3456ab43407ad3d372689 2024-11-21T00:27:55,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a7af240c11524c96ad00db0089a1b74f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a7af240c11524c96ad00db0089a1b74f 2024-11-21T00:27:55,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7f928af45184487cb36cd91256a5b1d9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7f928af45184487cb36cd91256a5b1d9 2024-11-21T00:27:55,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9022793c0fe445b4b80d829265ef0d91 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9022793c0fe445b4b80d829265ef0d91 2024-11-21T00:27:55,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/af4efc562bb04d80bd114f33198b87b9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/af4efc562bb04d80bd114f33198b87b9 2024-11-21T00:27:55,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b67d266f09e24bd0ad8d03e056c8ebcb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b67d266f09e24bd0ad8d03e056c8ebcb 2024-11-21T00:27:55,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f0ed2f03b46e4796b855d4d774dde8d4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f0ed2f03b46e4796b855d4d774dde8d4 2024-11-21T00:27:55,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a4097c4bed4e4c2b94929e0511e49586 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/a4097c4bed4e4c2b94929e0511e49586 2024-11-21T00:27:55,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/fe9d9799c0954309ac1713a7de0b57aa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/fe9d9799c0954309ac1713a7de0b57aa 2024-11-21T00:27:55,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b0d031c1b4f24313a186d32f07ec504c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/b0d031c1b4f24313a186d32f07ec504c 2024-11-21T00:27:55,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0b0aae8c9244d21bd11f8cf7add1144 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/c0b0aae8c9244d21bd11f8cf7add1144 2024-11-21T00:27:55,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7b9c1afaa8d04d5fb9b78583fdd57424 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/7b9c1afaa8d04d5fb9b78583fdd57424 2024-11-21T00:27:55,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/58b95f2855314032ade6723525983829 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/58b95f2855314032ade6723525983829 2024-11-21T00:27:55,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/dfb2b31059354dfe8839c579353c3b82 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/dfb2b31059354dfe8839c579353c3b82 2024-11-21T00:27:55,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/172f06bcc3da46fb890883665f4e90d4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/172f06bcc3da46fb890883665f4e90d4 2024-11-21T00:27:55,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba158e0fe6b745419a8ca2a1282a4b3f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba158e0fe6b745419a8ca2a1282a4b3f 2024-11-21T00:27:55,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/0f906a03bb334adcb2d9903fdf8d63cb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/0f906a03bb334adcb2d9903fdf8d63cb 2024-11-21T00:27:55,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/2269e096b48443fe95d6a10ceca5ea80 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/2269e096b48443fe95d6a10ceca5ea80 2024-11-21T00:27:55,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/e5be784a7a3447c3a89412909a24100c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/e5be784a7a3447c3a89412909a24100c 2024-11-21T00:27:55,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/8016b48b25f142e1a97d2f045bbb0d6d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/8016b48b25f142e1a97d2f045bbb0d6d 2024-11-21T00:27:55,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba79e52bd8194176994861b54723f3a3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/ba79e52bd8194176994861b54723f3a3 2024-11-21T00:27:55,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f8a0e056295f434fb0372f05357e7de1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f8a0e056295f434fb0372f05357e7de1 2024-11-21T00:27:55,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/1800fe1ae27b4539982d02157e6eba04 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/1800fe1ae27b4539982d02157e6eba04 2024-11-21T00:27:55,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/022aab94d50e4fe4a84270bb5dd6bd2c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/022aab94d50e4fe4a84270bb5dd6bd2c 2024-11-21T00:27:55,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/728edee8974b40a8877421713f379a7e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/728edee8974b40a8877421713f379a7e 2024-11-21T00:27:55,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f1baa4ca2b7d42f0a2fff42728318fe2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/f1baa4ca2b7d42f0a2fff42728318fe2 2024-11-21T00:27:55,936 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/recovered.edits/518.seqid, newMaxSeqId=518, maxSeqId=1 2024-11-21T00:27:55,939 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. 2024-11-21T00:27:55,940 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 9f0e4f53506bf5b3f736664c2080ec6c: Waiting for close lock at 1732148874498Running coprocessor pre-close hooks at 1732148874498Disabling compacts and flushes for region at 1732148874498Disabling writes for close at 1732148874499 (+1 ms)Obtaining lock to block concurrent updates at 1732148874499Preparing flush snapshotting stores in 9f0e4f53506bf5b3f736664c2080ec6c at 1732148874499Finished memstore snapshotting TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732148874499Flushing stores of TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c. at 1732148874502 (+3 ms)Flushing 9f0e4f53506bf5b3f736664c2080ec6c/A: creating writer at 1732148874502Flushing 9f0e4f53506bf5b3f736664c2080ec6c/A: appending metadata at 1732148874505 (+3 ms)Flushing 9f0e4f53506bf5b3f736664c2080ec6c/A: closing flushed file at 1732148874505Flushing 9f0e4f53506bf5b3f736664c2080ec6c/B: creating writer at 1732148874919 (+414 ms)Flushing 9f0e4f53506bf5b3f736664c2080ec6c/B: appending metadata at 1732148874921 (+2 ms)Flushing 9f0e4f53506bf5b3f736664c2080ec6c/B: closing flushed file at 1732148874921Flushing 9f0e4f53506bf5b3f736664c2080ec6c/C: creating writer at 1732148875329 (+408 ms)Flushing 9f0e4f53506bf5b3f736664c2080ec6c/C: appending metadata at 1732148875332 (+3 ms)Flushing 9f0e4f53506bf5b3f736664c2080ec6c/C: closing flushed file at 1732148875332Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53d7ba21: reopening flushed file at 1732148875738 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9654e8c: reopening flushed file at 1732148875743 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b419806: reopening flushed file at 1732148875748 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9f0e4f53506bf5b3f736664c2080ec6c in 1254ms, sequenceid=515, compaction requested=false at 1732148875753 (+5 ms)Writing region close event to WAL at 1732148875920 (+167 ms)Running coprocessor post-close hooks at 1732148875939 (+19 ms)Closed at 1732148875939 2024-11-21T00:27:55,949 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:55,949 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=9f0e4f53506bf5b3f736664c2080ec6c, regionState=CLOSED 2024-11-21T00:27:55,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:27:55,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T00:27:55,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 9f0e4f53506bf5b3f736664c2080ec6c, server=5ed4808ef0e6,38589,1732148778087 in 1.6190 sec 2024-11-21T00:27:55,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-21T00:27:55,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f0e4f53506bf5b3f736664c2080ec6c, UNASSIGN in 1.6240 sec 2024-11-21T00:27:55,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T00:27:55,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6290 sec 2024-11-21T00:27:55,974 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148875974"}]},"ts":"1732148875974"} 2024-11-21T00:27:55,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:27:55,977 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:27:55,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6420 sec 2024-11-21T00:27:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T00:27:56,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:56,482 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:27:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,484 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:27:56,485 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,487 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:56,489 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/recovered.edits] 2024-11-21T00:27:56,493 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/1efd67af34dc4c9f862f46c3a8c2d235 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/1efd67af34dc4c9f862f46c3a8c2d235 2024-11-21T00:27:56,494 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/375f7725dfe94f72a7511600ff6a0c48 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/A/375f7725dfe94f72a7511600ff6a0c48 2024-11-21T00:27:56,496 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0da9ec6355ac4659ad803ea66ce5061b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/0da9ec6355ac4659ad803ea66ce5061b 2024-11-21T00:27:56,498 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8aec8843954f4e1e81f6c9c38ff7223f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/B/8aec8843954f4e1e81f6c9c38ff7223f 2024-11-21T00:27:56,501 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/18bc2edfaa774a87ac66b6da9dc9697e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/18bc2edfaa774a87ac66b6da9dc9697e 2024-11-21T00:27:56,510 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9d2e92079b7b4a49ad23a373217b574a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/C/9d2e92079b7b4a49ad23a373217b574a 2024-11-21T00:27:56,513 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/recovered.edits/518.seqid to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c/recovered.edits/518.seqid 2024-11-21T00:27:56,514 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/9f0e4f53506bf5b3f736664c2080ec6c 2024-11-21T00:27:56,514 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:27:56,518 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,521 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:27:56,523 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:27:56,524 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,525 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:27:56,525 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148876525"}]},"ts":"9223372036854775807"} 2024-11-21T00:27:56,528 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:27:56,528 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9f0e4f53506bf5b3f736664c2080ec6c, NAME => 'TestAcidGuarantees,,1732148844917.9f0e4f53506bf5b3f736664c2080ec6c.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:27:56,528 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:27:56,528 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148876528"}]},"ts":"9223372036854775807"} 2024-11-21T00:27:56,531 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:27:56,532 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-21T00:27:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T00:27:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:27:56,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:56,604 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 241), OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=991 (was 997), ProcessCount=11 (was 11), AvailableMemoryMB=266 (was 727) 2024-11-21T00:27:56,615 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=991, ProcessCount=11, AvailableMemoryMB=265 2024-11-21T00:27:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:27:56,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:27:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:56,619 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:27:56,619 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:56,619 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-21T00:27:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-21T00:27:56,620 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:27:56,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742169_1345 (size=959) 2024-11-21T00:27:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-21T00:27:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-21T00:27:57,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:27:57,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742170_1346 (size=53) 2024-11-21T00:27:57,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:27:57,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5776037b580ebf80c0ab7fa6d7225dd4, disabling compactions & flushes 2024-11-21T00:27:57,040 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. after waiting 0 ms 2024-11-21T00:27:57,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,040 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5776037b580ebf80c0ab7fa6d7225dd4: Waiting for close lock at 1732148877040Disabling compacts and flushes for region at 1732148877040Disabling writes for close at 1732148877040Writing region close event to WAL at 1732148877040Closed at 1732148877040 2024-11-21T00:27:57,042 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:27:57,042 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148877042"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148877042"}]},"ts":"1732148877042"} 2024-11-21T00:27:57,044 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:27:57,045 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:27:57,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148877045"}]},"ts":"1732148877045"} 2024-11-21T00:27:57,047 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:27:57,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, ASSIGN}] 2024-11-21T00:27:57,051 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, ASSIGN 2024-11-21T00:27:57,052 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=false 2024-11-21T00:27:57,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:57,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, ASSIGN because future has completed 2024-11-21T00:27:57,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:27:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-21T00:27:57,358 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,358 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:27:57,359 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,359 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:27:57,359 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,359 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,360 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,361 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5776037b580ebf80c0ab7fa6d7225dd4 columnFamilyName A 2024-11-21T00:27:57,361 DEBUG [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:57,362 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(327): Store=5776037b580ebf80c0ab7fa6d7225dd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:57,362 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,363 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5776037b580ebf80c0ab7fa6d7225dd4 columnFamilyName B 2024-11-21T00:27:57,363 DEBUG [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:57,363 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(327): Store=5776037b580ebf80c0ab7fa6d7225dd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:57,363 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,364 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5776037b580ebf80c0ab7fa6d7225dd4 columnFamilyName C 2024-11-21T00:27:57,364 DEBUG [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:57,364 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(327): Store=5776037b580ebf80c0ab7fa6d7225dd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:57,365 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,365 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,365 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,366 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,366 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,367 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:27:57,367 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,369 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:27:57,370 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 5776037b580ebf80c0ab7fa6d7225dd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70379674, jitterRate=0.04873886704444885}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:27:57,370 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,370 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 5776037b580ebf80c0ab7fa6d7225dd4: Running coprocessor pre-open hook at 1732148877359Writing region info on filesystem at 1732148877359Initializing all the Stores at 1732148877360 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148877360Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148877360Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148877360Cleaning up temporary data from old regions at 1732148877366 (+6 ms)Running coprocessor post-open hooks at 1732148877370 (+4 ms)Region opened successfully at 1732148877370 2024-11-21T00:27:57,371 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., pid=85, masterSystemTime=1732148877355 2024-11-21T00:27:57,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:57,374 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,374 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:27:57,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T00:27:57,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 in 181 msec 2024-11-21T00:27:57,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-21T00:27:57,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, ASSIGN in 341 msec 2024-11-21T00:27:57,391 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:27:57,391 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148877391"}]},"ts":"1732148877391"} 2024-11-21T00:27:57,394 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:27:57,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:27:57,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 778 msec 2024-11-21T00:27:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-21T00:27:57,753 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:57,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ab9835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:57,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:57,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:57,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e65acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:57,759 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656f0170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:57,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:57,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:57,763 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:27:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:27:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@338df2e9 2024-11-21T00:27:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:27:57,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:27:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:27:57,771 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:27:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T00:27:57,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742171_1347 (size=995) 2024-11-21T00:27:57,800 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T00:27:57,800 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T00:27:57,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:27:57,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, REOPEN/MOVE}] 2024-11-21T00:27:57,809 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, REOPEN/MOVE 2024-11-21T00:27:57,810 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:57,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, REOPEN/MOVE because future has completed 2024-11-21T00:27:57,812 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:27:57,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:27:57,965 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,965 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:27:57,965 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 5776037b580ebf80c0ab7fa6d7225dd4, disabling compactions & flushes 2024-11-21T00:27:57,965 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,965 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,965 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. after waiting 0 ms 2024-11-21T00:27:57,965 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,971 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T00:27:57,971 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:57,971 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 5776037b580ebf80c0ab7fa6d7225dd4: Waiting for close lock at 1732148877965Running coprocessor pre-close hooks at 1732148877965Disabling compacts and flushes for region at 1732148877965Disabling writes for close at 1732148877965Writing region close event to WAL at 1732148877968 (+3 ms)Running coprocessor post-close hooks at 1732148877971 (+3 ms)Closed at 1732148877971 2024-11-21T00:27:57,971 WARN [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 5776037b580ebf80c0ab7fa6d7225dd4 to self. 2024-11-21T00:27:57,973 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:57,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=CLOSED 2024-11-21T00:27:57,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:27:57,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-21T00:27:57,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 in 163 msec 2024-11-21T00:27:57,977 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, REOPEN/MOVE; state=CLOSED, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=true 2024-11-21T00:27:58,127 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, REOPEN/MOVE because future has completed 2024-11-21T00:27:58,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:27:58,283 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,283 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:27:58,283 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,283 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:27:58,283 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,283 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,284 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,285 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5776037b580ebf80c0ab7fa6d7225dd4 columnFamilyName A 2024-11-21T00:27:58,286 DEBUG [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:58,286 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(327): Store=5776037b580ebf80c0ab7fa6d7225dd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:58,287 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,287 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5776037b580ebf80c0ab7fa6d7225dd4 columnFamilyName B 2024-11-21T00:27:58,287 DEBUG [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:58,288 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(327): Store=5776037b580ebf80c0ab7fa6d7225dd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:58,288 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,288 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5776037b580ebf80c0ab7fa6d7225dd4 columnFamilyName C 2024-11-21T00:27:58,288 DEBUG [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:58,289 INFO [StoreOpener-5776037b580ebf80c0ab7fa6d7225dd4-1 {}] regionserver.HStore(327): Store=5776037b580ebf80c0ab7fa6d7225dd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:27:58,289 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,289 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,290 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,291 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,291 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,291 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:27:58,292 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,293 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 5776037b580ebf80c0ab7fa6d7225dd4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59090401, jitterRate=-0.11948440968990326}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:27:58,293 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,294 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 5776037b580ebf80c0ab7fa6d7225dd4: Running coprocessor pre-open hook at 1732148878283Writing region info on filesystem at 1732148878283Initializing all the Stores at 1732148878284 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148878284Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148878284Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148878284Cleaning up temporary data from old regions at 1732148878291 (+7 ms)Running coprocessor post-open hooks at 1732148878293 (+2 ms)Region opened successfully at 1732148878294 (+1 ms) 2024-11-21T00:27:58,294 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., pid=90, masterSystemTime=1732148878280 2024-11-21T00:27:58,296 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,296 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=OPEN, openSeqNum=5, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:27:58,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-21T00:27:58,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 in 186 msec 2024-11-21T00:27:58,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-21T00:27:58,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, REOPEN/MOVE in 508 msec 2024-11-21T00:27:58,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T00:27:58,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-21T00:27:58,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 548 msec 2024-11-21T00:27:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T00:27:58,322 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:58,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@495e8dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75261e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,328 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2246ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,329 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f783c91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422bf042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d921ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,337 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4aadf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,339 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1daabe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,340 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6f770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,341 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231b7e9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,343 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ef8bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,344 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4303c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,346 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6462802d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,347 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149275cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a03bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,350 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9e1aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,352 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2237761f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30cea5e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:27:58,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:27:58,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:27:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:27:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:27:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea7c515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:27:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:27:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:27:58,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:27:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1ac765, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:27:58,364 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,364 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,364 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,365 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,365 DEBUG [Thread-1583 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,365 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,366 DEBUG [Thread-1583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,368 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,368 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,380 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:58,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,388 DEBUG [Thread-1587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,390 DEBUG [Thread-1587 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,390 DEBUG [Thread-1587 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,396 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:27:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:27:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:27:58,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:27:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:27:58,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-21T00:27:58,411 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T00:27:58,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:58,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:58,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b387f28417f04496ab33df93da85eccc_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148878384/Put/seqid=0 2024-11-21T00:27:58,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742172_1348 (size=12154) 2024-11-21T00:27:58,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:58,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b387f28417f04496ab33df93da85eccc_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b387f28417f04496ab33df93da85eccc_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3d973ccba3df4636abe9df9ae7f40fa8, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:58,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3d973ccba3df4636abe9df9ae7f40fa8 is 175, key is test_row_0/A:col10/1732148878384/Put/seqid=0 2024-11-21T00:27:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148938446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148938449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148938449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148938451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148938453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742173_1349 (size=30955) 2024-11-21T00:27:58,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3d973ccba3df4636abe9df9ae7f40fa8 2024-11-21T00:27:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T00:27:58,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f973e4cdf4d244c3acecd845ccd2b976 is 50, key is test_row_0/B:col10/1732148878384/Put/seqid=0 2024-11-21T00:27:58,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T00:27:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:58,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742174_1350 (size=12001) 2024-11-21T00:27:58,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148938572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148938572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f973e4cdf4d244c3acecd845ccd2b976 2024-11-21T00:27:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148938573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148938573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148938573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/06c73f682cb74fc6b5e91a4619d0ef43 is 50, key is test_row_0/C:col10/1732148878384/Put/seqid=0 2024-11-21T00:27:58,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742175_1351 (size=12001) 2024-11-21T00:27:58,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/06c73f682cb74fc6b5e91a4619d0ef43 2024-11-21T00:27:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3d973ccba3df4636abe9df9ae7f40fa8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8 2024-11-21T00:27:58,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8, entries=150, sequenceid=15, filesize=30.2 K 2024-11-21T00:27:58,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f973e4cdf4d244c3acecd845ccd2b976 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f973e4cdf4d244c3acecd845ccd2b976 2024-11-21T00:27:58,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f973e4cdf4d244c3acecd845ccd2b976, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T00:27:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/06c73f682cb74fc6b5e91a4619d0ef43 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/06c73f682cb74fc6b5e91a4619d0ef43 2024-11-21T00:27:58,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/06c73f682cb74fc6b5e91a4619d0ef43, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T00:27:58,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5776037b580ebf80c0ab7fa6d7225dd4 in 294ms, sequenceid=15, compaction requested=false 2024-11-21T00:27:58,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:58,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T00:27:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,719 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:27:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T00:27:58,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211438a24e6107464d976babcc7d076e3e_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148878434/Put/seqid=0 2024-11-21T00:27:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742176_1352 (size=12154) 2024-11-21T00:27:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:58,786 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211438a24e6107464d976babcc7d076e3e_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211438a24e6107464d976babcc7d076e3e_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/190767549a164c7582f3103d05cc87bd, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/190767549a164c7582f3103d05cc87bd is 175, key is test_row_0/A:col10/1732148878434/Put/seqid=0 2024-11-21T00:27:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:58,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148938802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742177_1353 (size=30955) 2024-11-21T00:27:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,818 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/190767549a164c7582f3103d05cc87bd 2024-11-21T00:27:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148938814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148938814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148938817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148938817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:58,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8c1db950410c448596eebbf2bbb6876c is 50, key is test_row_0/B:col10/1732148878434/Put/seqid=0 2024-11-21T00:27:58,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742178_1354 (size=12001) 2024-11-21T00:27:58,858 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8c1db950410c448596eebbf2bbb6876c 2024-11-21T00:27:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e226072c28eb40dfbb926f90bfe8a666 is 50, key is test_row_0/C:col10/1732148878434/Put/seqid=0 2024-11-21T00:27:58,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742179_1355 (size=12001) 2024-11-21T00:27:58,888 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e226072c28eb40dfbb926f90bfe8a666 2024-11-21T00:27:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/190767549a164c7582f3103d05cc87bd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd 2024-11-21T00:27:58,897 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T00:27:58,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8c1db950410c448596eebbf2bbb6876c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8c1db950410c448596eebbf2bbb6876c 2024-11-21T00:27:58,905 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8c1db950410c448596eebbf2bbb6876c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T00:27:58,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e226072c28eb40dfbb926f90bfe8a666 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e226072c28eb40dfbb926f90bfe8a666 2024-11-21T00:27:58,914 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e226072c28eb40dfbb926f90bfe8a666, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T00:27:58,915 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 5776037b580ebf80c0ab7fa6d7225dd4 in 196ms, sequenceid=40, compaction requested=false 2024-11-21T00:27:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-21T00:27:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-21T00:27:58,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-21T00:27:58,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 508 msec 2024-11-21T00:27:58,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 515 msec 2024-11-21T00:27:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:58,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:27:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a893dbda0a20408e8544194c160379c6_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:58,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742180_1356 (size=14594) 2024-11-21T00:27:58,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:58,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a893dbda0a20408e8544194c160379c6_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a893dbda0a20408e8544194c160379c6_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:58,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/11364b0efa47416cb2a15a36ceb1186f, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:58,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/11364b0efa47416cb2a15a36ceb1186f is 175, key is test_row_0/A:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:58,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742181_1357 (size=39549) 2024-11-21T00:27:58,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/11364b0efa47416cb2a15a36ceb1186f 2024-11-21T00:27:59,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/1295cccdab8e4291914a62820b883115 is 50, key is test_row_0/B:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148938994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148938994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148938997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148939009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148939011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742182_1358 (size=12001) 2024-11-21T00:27:59,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/1295cccdab8e4291914a62820b883115 2024-11-21T00:27:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T00:27:59,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:27:59,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:27:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-21T00:27:59,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5b88c35d739f43c1a2e19032a69b51cd is 50, key is test_row_0/C:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:59,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:27:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:27:59,047 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:27:59,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:27:59,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742183_1359 (size=12001) 2024-11-21T00:27:59,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5b88c35d739f43c1a2e19032a69b51cd 2024-11-21T00:27:59,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/11364b0efa47416cb2a15a36ceb1186f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f 2024-11-21T00:27:59,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f, entries=200, sequenceid=53, filesize=38.6 K 2024-11-21T00:27:59,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/1295cccdab8e4291914a62820b883115 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1295cccdab8e4291914a62820b883115 2024-11-21T00:27:59,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1295cccdab8e4291914a62820b883115, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T00:27:59,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5b88c35d739f43c1a2e19032a69b51cd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5b88c35d739f43c1a2e19032a69b51cd 2024-11-21T00:27:59,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5b88c35d739f43c1a2e19032a69b51cd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T00:27:59,092 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T00:27:59,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5776037b580ebf80c0ab7fa6d7225dd4 in 163ms, sequenceid=53, compaction requested=true 2024-11-21T00:27:59,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:59,093 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:59,094 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:59,094 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:27:59,094 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,094 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=99.1 K 2024-11-21T00:27:59,095 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,095 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f] 2024-11-21T00:27:59,095 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d973ccba3df4636abe9df9ae7f40fa8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148878378 2024-11-21T00:27:59,095 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 190767549a164c7582f3103d05cc87bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732148878434 2024-11-21T00:27:59,095 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11364b0efa47416cb2a15a36ceb1186f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148878800 2024-11-21T00:27:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:27:59,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:59,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:27:59,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:59,096 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:59,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:27:59,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:59,097 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:59,097 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:27:59,097 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,098 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f973e4cdf4d244c3acecd845ccd2b976, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8c1db950410c448596eebbf2bbb6876c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1295cccdab8e4291914a62820b883115] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=35.2 K 2024-11-21T00:27:59,098 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f973e4cdf4d244c3acecd845ccd2b976, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148878378 2024-11-21T00:27:59,098 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c1db950410c448596eebbf2bbb6876c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732148878434 2024-11-21T00:27:59,099 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1295cccdab8e4291914a62820b883115, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148878811 2024-11-21T00:27:59,113 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:59,126 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:59,127 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/965417276d0e4b41ae8c061d4e27765f is 50, key is test_row_0/B:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:27:59,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:27:59,133 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f5e1a965ffc1455ca77d714649ff84e0_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:59,136 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f5e1a965ffc1455ca77d714649ff84e0_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:59,136 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f5e1a965ffc1455ca77d714649ff84e0_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:27:59,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eb8bd63a4a5244e790975a61cd10a818_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148879008/Put/seqid=0 2024-11-21T00:27:59,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148939144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148939145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742184_1360 (size=12104) 2024-11-21T00:27:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148939158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148939159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,169 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/965417276d0e4b41ae8c061d4e27765f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/965417276d0e4b41ae8c061d4e27765f 2024-11-21T00:27:59,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148939161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,176 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into 965417276d0e4b41ae8c061d4e27765f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:59,176 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:59,177 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=13, startTime=1732148879096; duration=0sec 2024-11-21T00:27:59,177 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:27:59,177 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:27:59,177 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:27:59,178 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:27:59,178 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:27:59,178 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,178 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/06c73f682cb74fc6b5e91a4619d0ef43, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e226072c28eb40dfbb926f90bfe8a666, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5b88c35d739f43c1a2e19032a69b51cd] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=35.2 K 2024-11-21T00:27:59,178 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c73f682cb74fc6b5e91a4619d0ef43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148878378 2024-11-21T00:27:59,179 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e226072c28eb40dfbb926f90bfe8a666, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732148878434 2024-11-21T00:27:59,179 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b88c35d739f43c1a2e19032a69b51cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148878811 2024-11-21T00:27:59,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742185_1361 (size=4469) 2024-11-21T00:27:59,192 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#305 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:59,192 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/36d88c536518438ea9c497b16d5ae5b2 is 175, key is test_row_0/A:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:59,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T00:27:59,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:59,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,214 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:27:59,215 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/9d136a9af266416fa7d886319d60e56e is 50, key is test_row_0/C:col10/1732148878926/Put/seqid=0 2024-11-21T00:27:59,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742186_1362 (size=12154) 2024-11-21T00:27:59,221 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:27:59,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121eb8bd63a4a5244e790975a61cd10a818_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eb8bd63a4a5244e790975a61cd10a818_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:27:59,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/30b25fdf27fd4bfc813700e7dd4951d0, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:27:59,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/30b25fdf27fd4bfc813700e7dd4951d0 is 175, key is test_row_0/A:col10/1732148879008/Put/seqid=0 2024-11-21T00:27:59,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742187_1363 (size=31058) 2024-11-21T00:27:59,259 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/36d88c536518438ea9c497b16d5ae5b2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/36d88c536518438ea9c497b16d5ae5b2 2024-11-21T00:27:59,265 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into 36d88c536518438ea9c497b16d5ae5b2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:59,265 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:59,265 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=13, startTime=1732148879093; duration=0sec 2024-11-21T00:27:59,265 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:59,265 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:27:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148939273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148939273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148939273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148939273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148939282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742188_1364 (size=12104) 2024-11-21T00:27:59,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742189_1365 (size=30955) 2024-11-21T00:27:59,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/30b25fdf27fd4bfc813700e7dd4951d0 2024-11-21T00:27:59,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/126a652f6f834fc3a5ada30211649e42 is 50, key is test_row_0/B:col10/1732148879008/Put/seqid=0 2024-11-21T00:27:59,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T00:27:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:27:59,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742190_1366 (size=12001) 2024-11-21T00:27:59,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/126a652f6f834fc3a5ada30211649e42 2024-11-21T00:27:59,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/641cf3988b914dc3b09dba40e7a08296 is 50, key is test_row_0/C:col10/1732148879008/Put/seqid=0 2024-11-21T00:27:59,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742191_1367 (size=12001) 2024-11-21T00:27:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148939492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148939492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148939492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148939493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T00:27:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:59,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148939493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T00:27:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:27:59,717 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/9d136a9af266416fa7d886319d60e56e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/9d136a9af266416fa7d886319d60e56e 2024-11-21T00:27:59,721 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into 9d136a9af266416fa7d886319d60e56e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:27:59,721 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:59,721 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=13, startTime=1732148879096; duration=0sec 2024-11-21T00:27:59,721 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:27:59,722 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:27:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T00:27:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:27:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:27:59,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148939813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148939813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148939813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148939817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:27:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148939822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:27:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:27:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:27:59,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/641cf3988b914dc3b09dba40e7a08296 2024-11-21T00:27:59,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/30b25fdf27fd4bfc813700e7dd4951d0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0 2024-11-21T00:27:59,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0, entries=150, sequenceid=78, filesize=30.2 K 2024-11-21T00:27:59,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/126a652f6f834fc3a5ada30211649e42 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/126a652f6f834fc3a5ada30211649e42 2024-11-21T00:27:59,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/126a652f6f834fc3a5ada30211649e42, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T00:27:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/641cf3988b914dc3b09dba40e7a08296 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/641cf3988b914dc3b09dba40e7a08296 2024-11-21T00:27:59,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/641cf3988b914dc3b09dba40e7a08296, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T00:27:59,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5776037b580ebf80c0ab7fa6d7225dd4 in 742ms, sequenceid=78, compaction requested=false 2024-11-21T00:27:59,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:27:59,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T00:27:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:27:59,976 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:27:59,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112162a3dfca8d5243cd8338758ba3dcb5ef_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148879156/Put/seqid=0 2024-11-21T00:28:00,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742192_1368 (size=12154) 2024-11-21T00:28:00,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:00,036 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112162a3dfca8d5243cd8338758ba3dcb5ef_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112162a3dfca8d5243cd8338758ba3dcb5ef_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d1ce90361ef54dfa8eed78f089ec91af, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d1ce90361ef54dfa8eed78f089ec91af is 175, key is test_row_0/A:col10/1732148879156/Put/seqid=0 2024-11-21T00:28:00,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742193_1369 (size=30955) 2024-11-21T00:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:28:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:00,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148940442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,470 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d1ce90361ef54dfa8eed78f089ec91af 2024-11-21T00:28:00,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148940454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148940454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148940455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148940456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5e92584dae824cbd86e8cab328e102df is 50, key is test_row_0/B:col10/1732148879156/Put/seqid=0 2024-11-21T00:28:00,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742194_1370 (size=12001) 2024-11-21T00:28:00,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148940562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148940582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148940582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148940582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148940582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148940772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148940805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148940805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148940805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148940805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:00,930 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5e92584dae824cbd86e8cab328e102df 2024-11-21T00:28:00,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/a00b0e109f824b85b2b7f79c3ee799a4 is 50, key is test_row_0/C:col10/1732148879156/Put/seqid=0 2024-11-21T00:28:01,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742195_1371 (size=12001) 2024-11-21T00:28:01,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148941092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148941122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148941122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148941122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148941123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:28:01,413 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/a00b0e109f824b85b2b7f79c3ee799a4 2024-11-21T00:28:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d1ce90361ef54dfa8eed78f089ec91af as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af 2024-11-21T00:28:01,423 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af, entries=150, sequenceid=92, filesize=30.2 K 2024-11-21T00:28:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5e92584dae824cbd86e8cab328e102df as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5e92584dae824cbd86e8cab328e102df 2024-11-21T00:28:01,428 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5e92584dae824cbd86e8cab328e102df, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T00:28:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/a00b0e109f824b85b2b7f79c3ee799a4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a00b0e109f824b85b2b7f79c3ee799a4 2024-11-21T00:28:01,433 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a00b0e109f824b85b2b7f79c3ee799a4, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T00:28:01,434 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5776037b580ebf80c0ab7fa6d7225dd4 in 1458ms, sequenceid=92, compaction requested=true 2024-11-21T00:28:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-21T00:28:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-21T00:28:01,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T00:28:01,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3900 sec 2024-11-21T00:28:01,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3950 sec 2024-11-21T00:28:01,479 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:28:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:01,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:28:01,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121233b30e578734ee7a7a3479d67831ce7_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:01,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148941660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148941660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148941662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148941665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742196_1372 (size=14594) 2024-11-21T00:28:01,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:01,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148941668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121233b30e578734ee7a7a3479d67831ce7_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121233b30e578734ee7a7a3479d67831ce7_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:01,692 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/97aa0d5e14cb44bab4ebaca45a189499, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:01,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/97aa0d5e14cb44bab4ebaca45a189499 is 175, key is test_row_0/A:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:01,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742197_1373 (size=39549) 2024-11-21T00:28:01,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/97aa0d5e14cb44bab4ebaca45a189499 2024-11-21T00:28:01,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/6d9d11be3b454b5697fd9c0492693166 is 50, key is test_row_0/B:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:01,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148941782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148941782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148941782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148941782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148941792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:01,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742198_1374 (size=12001) 2024-11-21T00:28:01,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/6d9d11be3b454b5697fd9c0492693166 2024-11-21T00:28:01,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b8b3b063aefa40e7b6ba98acf8ad084b is 50, key is test_row_0/C:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:01,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742199_1375 (size=12001) 2024-11-21T00:28:02,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148942002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148942002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148942002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148942002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148942002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b8b3b063aefa40e7b6ba98acf8ad084b 2024-11-21T00:28:02,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/97aa0d5e14cb44bab4ebaca45a189499 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499 2024-11-21T00:28:02,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499, entries=200, sequenceid=118, filesize=38.6 K 2024-11-21T00:28:02,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/6d9d11be3b454b5697fd9c0492693166 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/6d9d11be3b454b5697fd9c0492693166 2024-11-21T00:28:02,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/6d9d11be3b454b5697fd9c0492693166, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:28:02,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b8b3b063aefa40e7b6ba98acf8ad084b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b8b3b063aefa40e7b6ba98acf8ad084b 2024-11-21T00:28:02,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b8b3b063aefa40e7b6ba98acf8ad084b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:28:02,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5776037b580ebf80c0ab7fa6d7225dd4 in 633ms, sequenceid=118, compaction requested=true 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:02,259 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:02,259 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:02,260 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:02,260 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:02,260 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:02,261 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/965417276d0e4b41ae8c061d4e27765f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/126a652f6f834fc3a5ada30211649e42, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5e92584dae824cbd86e8cab328e102df, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/6d9d11be3b454b5697fd9c0492693166] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=47.0 K 2024-11-21T00:28:02,261 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:02,261 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:02,261 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:02,261 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/36d88c536518438ea9c497b16d5ae5b2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=129.4 K 2024-11-21T00:28:02,261 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:02,261 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/36d88c536518438ea9c497b16d5ae5b2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499] 2024-11-21T00:28:02,261 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 965417276d0e4b41ae8c061d4e27765f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148878811 2024-11-21T00:28:02,264 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d88c536518438ea9c497b16d5ae5b2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148878811 2024-11-21T00:28:02,264 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 126a652f6f834fc3a5ada30211649e42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732148878971 2024-11-21T00:28:02,265 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b25fdf27fd4bfc813700e7dd4951d0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732148878971 2024-11-21T00:28:02,265 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e92584dae824cbd86e8cab328e102df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732148879143 2024-11-21T00:28:02,265 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d9d11be3b454b5697fd9c0492693166, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148880454 2024-11-21T00:28:02,265 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ce90361ef54dfa8eed78f089ec91af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732148879143 2024-11-21T00:28:02,266 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97aa0d5e14cb44bab4ebaca45a189499, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148880433 2024-11-21T00:28:02,281 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:02,283 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121e93f4b7f50b54acea89e366044dd93e3_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:02,285 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121e93f4b7f50b54acea89e366044dd93e3_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:02,285 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e93f4b7f50b54acea89e366044dd93e3_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:02,311 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:02,311 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/d24ab0aa764c45aeaf14ee5b770d536f is 50, key is test_row_0/B:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:02,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:02,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742200_1376 (size=4469) 2024-11-21T00:28:02,378 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#317 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:02,379 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/05e73b0a1e1148a5983936e4f27fffb0 is 175, key is test_row_0/A:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:02,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e175884c9b94475eba8bef673feb67e0_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148881658/Put/seqid=0 2024-11-21T00:28:02,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742201_1377 (size=12241) 2024-11-21T00:28:02,395 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/d24ab0aa764c45aeaf14ee5b770d536f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d24ab0aa764c45aeaf14ee5b770d536f 2024-11-21T00:28:02,401 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into d24ab0aa764c45aeaf14ee5b770d536f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:02,401 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:02,401 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=12, startTime=1732148882259; duration=0sec 2024-11-21T00:28:02,401 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:02,401 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:02,401 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:02,402 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:02,402 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:02,403 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:02,403 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/9d136a9af266416fa7d886319d60e56e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/641cf3988b914dc3b09dba40e7a08296, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a00b0e109f824b85b2b7f79c3ee799a4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b8b3b063aefa40e7b6ba98acf8ad084b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=47.0 K 2024-11-21T00:28:02,404 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d136a9af266416fa7d886319d60e56e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148878811 2024-11-21T00:28:02,404 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 641cf3988b914dc3b09dba40e7a08296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732148878971 2024-11-21T00:28:02,404 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a00b0e109f824b85b2b7f79c3ee799a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732148879143 2024-11-21T00:28:02,405 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b8b3b063aefa40e7b6ba98acf8ad084b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148880454 2024-11-21T00:28:02,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742202_1378 (size=31195) 2024-11-21T00:28:02,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148942401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148942406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148942407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,432 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/05e73b0a1e1148a5983936e4f27fffb0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/05e73b0a1e1148a5983936e4f27fffb0 2024-11-21T00:28:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148942412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,439 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into 05e73b0a1e1148a5983936e4f27fffb0(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:02,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:02,440 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=12, startTime=1732148882259; duration=0sec 2024-11-21T00:28:02,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:02,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:02,443 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:02,443 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/edafce3994c2432eaefff174cdfffed7 is 50, key is test_row_0/C:col10/1732148880454/Put/seqid=0 2024-11-21T00:28:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148942433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742203_1379 (size=14644) 2024-11-21T00:28:02,466 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:02,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e175884c9b94475eba8bef673feb67e0_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e175884c9b94475eba8bef673feb67e0_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:02,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/bd33c08429564818b87f5f912ac9ae0c, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:02,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/bd33c08429564818b87f5f912ac9ae0c is 175, key is test_row_0/A:col10/1732148881658/Put/seqid=0 2024-11-21T00:28:02,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742204_1380 (size=12241) 2024-11-21T00:28:02,531 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/edafce3994c2432eaefff174cdfffed7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/edafce3994c2432eaefff174cdfffed7 2024-11-21T00:28:02,536 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into edafce3994c2432eaefff174cdfffed7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:02,536 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:02,536 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=12, startTime=1732148882259; duration=0sec 2024-11-21T00:28:02,536 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:02,540 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148942533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148942542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148942542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148942543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742205_1381 (size=39599) 2024-11-21T00:28:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/bd33c08429564818b87f5f912ac9ae0c 2024-11-21T00:28:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148942563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/a99f962f0043451993ccfb092908e72a is 50, key is test_row_0/B:col10/1732148881658/Put/seqid=0 2024-11-21T00:28:02,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742206_1382 (size=12051) 2024-11-21T00:28:02,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148942752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148942762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148942762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148942763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:02,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148942772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/a99f962f0043451993ccfb092908e72a 2024-11-21T00:28:03,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47 is 50, key is test_row_0/C:col10/1732148881658/Put/seqid=0 2024-11-21T00:28:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742207_1383 (size=12051) 2024-11-21T00:28:03,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148943063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148943073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148943073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148943073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148943082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T00:28:03,228 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:03,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-21T00:28:03,238 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T00:28:03,239 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:03,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T00:28:03,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-21T00:28:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:03,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:03,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47 2024-11-21T00:28:03,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/bd33c08429564818b87f5f912ac9ae0c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c 2024-11-21T00:28:03,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c, entries=200, sequenceid=129, filesize=38.7 K 2024-11-21T00:28:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/a99f962f0043451993ccfb092908e72a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/a99f962f0043451993ccfb092908e72a 2024-11-21T00:28:03,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/a99f962f0043451993ccfb092908e72a, entries=150, sequenceid=129, filesize=11.8 K 2024-11-21T00:28:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47 2024-11-21T00:28:03,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47, entries=150, sequenceid=129, filesize=11.8 K 2024-11-21T00:28:03,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5776037b580ebf80c0ab7fa6d7225dd4 in 1173ms, sequenceid=129, compaction requested=false 2024-11-21T00:28:03,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:03,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-21T00:28:03,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:03,546 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:28:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T00:28:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d2f43f50a05a42549961778497205c86_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148882427/Put/seqid=0 2024-11-21T00:28:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:03,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:03,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148943619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148943620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742208_1384 (size=12304) 2024-11-21T00:28:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148943634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148943637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148943644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148943754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148943754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148943773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148943773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148943773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T00:28:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148943972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148943976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148943992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148943992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148944002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:04,046 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d2f43f50a05a42549961778497205c86_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d2f43f50a05a42549961778497205c86_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:04,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/96fb0b3f0e90451ab901633214e25c39, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:04,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/96fb0b3f0e90451ab901633214e25c39 is 175, key is test_row_0/A:col10/1732148882427/Put/seqid=0 2024-11-21T00:28:04,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742209_1385 (size=31105) 2024-11-21T00:28:04,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148944288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148944305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148944312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148944312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148944312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T00:28:04,495 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/96fb0b3f0e90451ab901633214e25c39 2024-11-21T00:28:04,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/844ce6ade75a4b9fb4a17b18e6117439 is 50, key is test_row_0/B:col10/1732148882427/Put/seqid=0 2024-11-21T00:28:04,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742210_1386 (size=12151) 2024-11-21T00:28:04,570 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/844ce6ade75a4b9fb4a17b18e6117439 2024-11-21T00:28:04,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/537be39614cc4fa1972c4ff18d7e3cb8 is 50, key is test_row_0/C:col10/1732148882427/Put/seqid=0 2024-11-21T00:28:04,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742211_1387 (size=12151) 2024-11-21T00:28:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148944822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148944822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148944833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148944834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148944835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:05,058 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/537be39614cc4fa1972c4ff18d7e3cb8 2024-11-21T00:28:05,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/96fb0b3f0e90451ab901633214e25c39 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39 2024-11-21T00:28:05,068 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39, entries=150, sequenceid=157, filesize=30.4 K 2024-11-21T00:28:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/844ce6ade75a4b9fb4a17b18e6117439 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/844ce6ade75a4b9fb4a17b18e6117439 2024-11-21T00:28:05,077 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/844ce6ade75a4b9fb4a17b18e6117439, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T00:28:05,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/537be39614cc4fa1972c4ff18d7e3cb8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/537be39614cc4fa1972c4ff18d7e3cb8 2024-11-21T00:28:05,085 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/537be39614cc4fa1972c4ff18d7e3cb8, entries=150, sequenceid=157, filesize=11.9 K 2024-11-21T00:28:05,087 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5776037b580ebf80c0ab7fa6d7225dd4 in 1541ms, sequenceid=157, compaction requested=true 2024-11-21T00:28:05,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:05,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:05,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-21T00:28:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-21T00:28:05,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-21T00:28:05,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-21T00:28:05,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8630 sec 2024-11-21T00:28:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T00:28:05,383 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:05,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-21T00:28:05,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:05,387 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:28:05,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:28:05,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T00:28:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:05,541 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213d8b48cb9fe34cfca51b283fb6247a8d_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148883632/Put/seqid=0 2024-11-21T00:28:05,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742212_1388 (size=12304) 2024-11-21T00:28:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:28:05,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148945928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:05,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148945928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:05,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148945929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:05,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148945933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:05,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148945933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:05,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:05,979 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213d8b48cb9fe34cfca51b283fb6247a8d_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213d8b48cb9fe34cfca51b283fb6247a8d_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/33aa7911c23c405f8396fa0287776433, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/33aa7911c23c405f8396fa0287776433 is 175, key is test_row_0/A:col10/1732148883632/Put/seqid=0 2024-11-21T00:28:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:28:06,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742213_1389 (size=31105) 2024-11-21T00:28:06,022 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/33aa7911c23c405f8396fa0287776433 2024-11-21T00:28:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/931258e261cc42c1b42867b0c3560d9c is 50, key is test_row_0/B:col10/1732148883632/Put/seqid=0 2024-11-21T00:28:06,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148946052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148946053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148946053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148946053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148946063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742214_1390 (size=12151) 2024-11-21T00:28:06,077 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/931258e261cc42c1b42867b0c3560d9c 2024-11-21T00:28:06,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/af44fa521a5d4af99d94280cc3a08d1d is 50, key is test_row_0/C:col10/1732148883632/Put/seqid=0 2024-11-21T00:28:06,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742215_1391 (size=12151) 2024-11-21T00:28:06,118 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/af44fa521a5d4af99d94280cc3a08d1d 2024-11-21T00:28:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/33aa7911c23c405f8396fa0287776433 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433 2024-11-21T00:28:06,133 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433, entries=150, sequenceid=168, filesize=30.4 K 2024-11-21T00:28:06,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/931258e261cc42c1b42867b0c3560d9c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/931258e261cc42c1b42867b0c3560d9c 2024-11-21T00:28:06,139 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/931258e261cc42c1b42867b0c3560d9c, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T00:28:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/af44fa521a5d4af99d94280cc3a08d1d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/af44fa521a5d4af99d94280cc3a08d1d 2024-11-21T00:28:06,146 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/af44fa521a5d4af99d94280cc3a08d1d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T00:28:06,147 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5776037b580ebf80c0ab7fa6d7225dd4 in 607ms, sequenceid=168, compaction requested=true 2024-11-21T00:28:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-21T00:28:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-21T00:28:06,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T00:28:06,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 762 msec 2024-11-21T00:28:06,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 767 msec 2024-11-21T00:28:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:06,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:06,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:28:06,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215dac43f559734e809b1cac56d94d80b0_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:06,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148946305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148946317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742216_1392 (size=12304) 2024-11-21T00:28:06,340 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148946324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148946324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148946324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,344 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215dac43f559734e809b1cac56d94d80b0_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215dac43f559734e809b1cac56d94d80b0_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:06,345 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/f3687d763348461fbf6f63a605b539a7, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:06,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/f3687d763348461fbf6f63a605b539a7 is 175, key is test_row_0/A:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:06,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742217_1393 (size=31105) 2024-11-21T00:28:06,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/f3687d763348461fbf6f63a605b539a7 2024-11-21T00:28:06,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/962fab00ba9349e3ad66d7bbf67f0223 is 50, key is test_row_0/B:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:06,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742218_1394 (size=12151) 2024-11-21T00:28:06,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/962fab00ba9349e3ad66d7bbf67f0223 2024-11-21T00:28:06,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e689490997684461a54de56aca24371a is 50, key is test_row_0/C:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:06,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742219_1395 (size=12151) 2024-11-21T00:28:06,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e689490997684461a54de56aca24371a 2024-11-21T00:28:06,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/f3687d763348461fbf6f63a605b539a7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7 2024-11-21T00:28:06,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7, entries=150, sequenceid=196, filesize=30.4 K 2024-11-21T00:28:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/962fab00ba9349e3ad66d7bbf67f0223 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/962fab00ba9349e3ad66d7bbf67f0223 2024-11-21T00:28:06,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/962fab00ba9349e3ad66d7bbf67f0223, entries=150, sequenceid=196, filesize=11.9 K 2024-11-21T00:28:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e689490997684461a54de56aca24371a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e689490997684461a54de56aca24371a 2024-11-21T00:28:06,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e689490997684461a54de56aca24371a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-21T00:28:06,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 5776037b580ebf80c0ab7fa6d7225dd4 in 154ms, sequenceid=196, compaction requested=true 2024-11-21T00:28:06,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:06,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:06,438 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:28:06,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:06,439 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:06,440 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60745 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:28:06,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164109 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:28:06,440 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:06,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:06,440 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,440 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,440 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d24ab0aa764c45aeaf14ee5b770d536f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/a99f962f0043451993ccfb092908e72a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/844ce6ade75a4b9fb4a17b18e6117439, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/931258e261cc42c1b42867b0c3560d9c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/962fab00ba9349e3ad66d7bbf67f0223] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=59.3 K 2024-11-21T00:28:06,440 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/05e73b0a1e1148a5983936e4f27fffb0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=160.3 K 2024-11-21T00:28:06,440 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/05e73b0a1e1148a5983936e4f27fffb0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7] 2024-11-21T00:28:06,440 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d24ab0aa764c45aeaf14ee5b770d536f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148880454 2024-11-21T00:28:06,441 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a99f962f0043451993ccfb092908e72a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732148881657 2024-11-21T00:28:06,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05e73b0a1e1148a5983936e4f27fffb0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148880454 2024-11-21T00:28:06,441 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 844ce6ade75a4b9fb4a17b18e6117439, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732148882401 2024-11-21T00:28:06,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd33c08429564818b87f5f912ac9ae0c, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732148881657 2024-11-21T00:28:06,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96fb0b3f0e90451ab901633214e25c39, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732148882401 2024-11-21T00:28:06,441 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 931258e261cc42c1b42867b0c3560d9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732148883600 2024-11-21T00:28:06,442 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 962fab00ba9349e3ad66d7bbf67f0223, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732148886281 2024-11-21T00:28:06,442 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33aa7911c23c405f8396fa0287776433, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732148883600 2024-11-21T00:28:06,443 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3687d763348461fbf6f63a605b539a7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732148886281 2024-11-21T00:28:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:06,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:28:06,469 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:06,469 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/82a50ab151c14cf2a79286a3cd42c8d8 is 50, key is test_row_0/B:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:06,478 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:06,493 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121e50063aa89bc4424b0f284b0a8af5a67_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:06,495 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121e50063aa89bc4424b0f284b0a8af5a67_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:06,496 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e50063aa89bc4424b0f284b0a8af5a67_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:06,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742220_1396 (size=12561) 2024-11-21T00:28:06,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121260e8f7c889c4d4e99065482ede03de6_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148886455/Put/seqid=0 2024-11-21T00:28:06,514 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/82a50ab151c14cf2a79286a3cd42c8d8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/82a50ab151c14cf2a79286a3cd42c8d8 2024-11-21T00:28:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T00:28:06,522 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:06,523 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into 82a50ab151c14cf2a79286a3cd42c8d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:06,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:06,523 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=11, startTime=1732148886438; duration=0sec 2024-11-21T00:28:06,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:06,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:06,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:06,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T00:28:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-21T00:28:06,524 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:28:06,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:06,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:06,527 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60745 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T00:28:06,527 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:06,527 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,527 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/edafce3994c2432eaefff174cdfffed7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/537be39614cc4fa1972c4ff18d7e3cb8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/af44fa521a5d4af99d94280cc3a08d1d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e689490997684461a54de56aca24371a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=59.3 K 2024-11-21T00:28:06,528 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting edafce3994c2432eaefff174cdfffed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148880454 2024-11-21T00:28:06,528 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e75e3e7b70e4dfaaaa2dc4ef8976f47, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732148881657 2024-11-21T00:28:06,528 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 537be39614cc4fa1972c4ff18d7e3cb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732148882401 2024-11-21T00:28:06,528 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting af44fa521a5d4af99d94280cc3a08d1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732148883600 2024-11-21T00:28:06,529 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e689490997684461a54de56aca24371a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732148886281 2024-11-21T00:28:06,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148946518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148946519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148946519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148946520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148946537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742222_1398 (size=19774) 2024-11-21T00:28:06,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742221_1397 (size=4469) 2024-11-21T00:28:06,561 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:06,567 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:06,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121260e8f7c889c4d4e99065482ede03de6_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121260e8f7c889c4d4e99065482ede03de6_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:06,568 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/2f142db6b0464af9a937e5eee3f389ae is 50, key is test_row_0/C:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:06,569 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d7688dd7394a419c819bba8e5764522e, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:06,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d7688dd7394a419c819bba8e5764522e is 175, key is test_row_0/A:col10/1732148886455/Put/seqid=0 2024-11-21T00:28:06,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742223_1399 (size=12561) 2024-11-21T00:28:06,588 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/2f142db6b0464af9a937e5eee3f389ae as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/2f142db6b0464af9a937e5eee3f389ae 2024-11-21T00:28:06,594 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into 2f142db6b0464af9a937e5eee3f389ae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:06,594 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:06,594 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=11, startTime=1732148886439; duration=0sec 2024-11-21T00:28:06,594 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:06,594 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:06,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742224_1400 (size=57033) 2024-11-21T00:28:06,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d7688dd7394a419c819bba8e5764522e 2024-11-21T00:28:06,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/c3a185cb69d14b698a3fcd4e53db4ee2 is 50, key is test_row_0/B:col10/1732148886455/Put/seqid=0 2024-11-21T00:28:06,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742225_1401 (size=12151) 2024-11-21T00:28:06,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/c3a185cb69d14b698a3fcd4e53db4ee2 2024-11-21T00:28:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:28:06,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/74ded48898594a94b54de3da4ea615a4 is 50, key is test_row_0/C:col10/1732148886455/Put/seqid=0 2024-11-21T00:28:06,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148946642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148946642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742226_1402 (size=12151) 2024-11-21T00:28:06,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/74ded48898594a94b54de3da4ea615a4 2024-11-21T00:28:06,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d7688dd7394a419c819bba8e5764522e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e 2024-11-21T00:28:06,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e, entries=300, sequenceid=210, filesize=55.7 K 2024-11-21T00:28:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/c3a185cb69d14b698a3fcd4e53db4ee2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/c3a185cb69d14b698a3fcd4e53db4ee2 2024-11-21T00:28:06,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148946662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148946662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148946662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/c3a185cb69d14b698a3fcd4e53db4ee2, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T00:28:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/74ded48898594a94b54de3da4ea615a4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/74ded48898594a94b54de3da4ea615a4 2024-11-21T00:28:06,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:28:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:06,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/74ded48898594a94b54de3da4ea615a4, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T00:28:06,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5776037b580ebf80c0ab7fa6d7225dd4 in 221ms, sequenceid=210, compaction requested=false 2024-11-21T00:28:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:06,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T00:28:06,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:06,832 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:28:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:28:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bb4d05f38017496d81e6119ba8c2fd36_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148886514/Put/seqid=0 2024-11-21T00:28:06,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742227_1403 (size=12304) 2024-11-21T00:28:06,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148946891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148946901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148946906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148946907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148946907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:06,962 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#333 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:06,963 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/15a9919f05f846a3b15886ceaa791e46 is 175, key is test_row_0/A:col10/1732148886283/Put/seqid=0 2024-11-21T00:28:07,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742228_1404 (size=31515) 2024-11-21T00:28:07,025 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/15a9919f05f846a3b15886ceaa791e46 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/15a9919f05f846a3b15886ceaa791e46 2024-11-21T00:28:07,031 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into 15a9919f05f846a3b15886ceaa791e46(size=30.8 K), total size for store is 86.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:07,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:07,031 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=11, startTime=1732148886438; duration=0sec 2024-11-21T00:28:07,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:07,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148947022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148947025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148947032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148947032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148947032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:28:07,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148947253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148947253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148947253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148947253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148947254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:07,319 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bb4d05f38017496d81e6119ba8c2fd36_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bb4d05f38017496d81e6119ba8c2fd36_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/90d93621476648ce82a54e400600b092, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/90d93621476648ce82a54e400600b092 is 175, key is test_row_0/A:col10/1732148886514/Put/seqid=0 2024-11-21T00:28:07,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742229_1405 (size=31105) 2024-11-21T00:28:07,334 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/90d93621476648ce82a54e400600b092 2024-11-21T00:28:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f91dd6bce8ac4623aa0794f50c41bfdc is 50, key is test_row_0/B:col10/1732148886514/Put/seqid=0 2024-11-21T00:28:07,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742230_1406 (size=12151) 2024-11-21T00:28:07,362 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f91dd6bce8ac4623aa0794f50c41bfdc 2024-11-21T00:28:07,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/46965fb3fc1d4c9c8b17c2ff568c4823 is 50, key is test_row_0/C:col10/1732148886514/Put/seqid=0 2024-11-21T00:28:07,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742231_1407 (size=12151) 2024-11-21T00:28:07,428 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/46965fb3fc1d4c9c8b17c2ff568c4823 2024-11-21T00:28:07,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/90d93621476648ce82a54e400600b092 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092 2024-11-21T00:28:07,437 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092, entries=150, sequenceid=234, filesize=30.4 K 2024-11-21T00:28:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f91dd6bce8ac4623aa0794f50c41bfdc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f91dd6bce8ac4623aa0794f50c41bfdc 2024-11-21T00:28:07,445 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f91dd6bce8ac4623aa0794f50c41bfdc, entries=150, sequenceid=234, filesize=11.9 K 2024-11-21T00:28:07,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/46965fb3fc1d4c9c8b17c2ff568c4823 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/46965fb3fc1d4c9c8b17c2ff568c4823 2024-11-21T00:28:07,450 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/46965fb3fc1d4c9c8b17c2ff568c4823, entries=150, sequenceid=234, filesize=11.9 K 2024-11-21T00:28:07,450 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5776037b580ebf80c0ab7fa6d7225dd4 in 619ms, sequenceid=234, compaction requested=true 2024-11-21T00:28:07,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:07,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:07,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-21T00:28:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-21T00:28:07,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T00:28:07,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 927 msec 2024-11-21T00:28:07,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 932 msec 2024-11-21T00:28:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:07,567 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:07,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T00:28:07,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d6a5a312f37b4e2ba28dcd0cabc590b6_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:07,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742232_1408 (size=14794) 2024-11-21T00:28:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148947636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148947639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148947639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148947640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148947641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T00:28:07,662 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:07,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T00:28:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:28:07,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:07,666 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:07,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148947752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148947752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148947752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148947752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148947753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:28:07,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:28:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:07,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:07,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:28:07,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:07,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:07,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:07,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:07,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:28:07,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,018 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:08,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d6a5a312f37b4e2ba28dcd0cabc590b6_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6a5a312f37b4e2ba28dcd0cabc590b6_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:08,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/a9c7dc17732643bda2d4aa01351a50a7, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:08,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/a9c7dc17732643bda2d4aa01351a50a7 is 175, key is test_row_0/A:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:08,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742233_1409 (size=39749) 2024-11-21T00:28:08,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/a9c7dc17732643bda2d4aa01351a50a7 2024-11-21T00:28:08,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/e40e944025b54cc78393235de7506078 is 50, key is test_row_0/B:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:08,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742234_1410 (size=12151) 2024-11-21T00:28:08,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/e40e944025b54cc78393235de7506078 2024-11-21T00:28:08,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:28:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:08,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/681991cbe39d4fa8b7881a49f4a5f7d9 is 50, key is test_row_0/C:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:08,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742235_1411 (size=12151) 2024-11-21T00:28:08,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/681991cbe39d4fa8b7881a49f4a5f7d9 2024-11-21T00:28:08,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/a9c7dc17732643bda2d4aa01351a50a7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7 2024-11-21T00:28:08,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7, entries=200, sequenceid=250, filesize=38.8 K 2024-11-21T00:28:08,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/e40e944025b54cc78393235de7506078 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/e40e944025b54cc78393235de7506078 2024-11-21T00:28:08,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/e40e944025b54cc78393235de7506078, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T00:28:08,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/681991cbe39d4fa8b7881a49f4a5f7d9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/681991cbe39d4fa8b7881a49f4a5f7d9 2024-11-21T00:28:08,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/681991cbe39d4fa8b7881a49f4a5f7d9, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T00:28:08,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5776037b580ebf80c0ab7fa6d7225dd4 in 662ms, sequenceid=250, compaction requested=true 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T00:28:08,229 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:08,229 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:08,231 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159402 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:08,231 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:08,231 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,231 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/15a9919f05f846a3b15886ceaa791e46, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=155.7 K 2024-11-21T00:28:08,231 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,231 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/15a9919f05f846a3b15886ceaa791e46, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7] 2024-11-21T00:28:08,231 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:08,231 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:08,231 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,231 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/2f142db6b0464af9a937e5eee3f389ae, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/74ded48898594a94b54de3da4ea615a4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/46965fb3fc1d4c9c8b17c2ff568c4823, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/681991cbe39d4fa8b7881a49f4a5f7d9] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=47.9 K 2024-11-21T00:28:08,232 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f142db6b0464af9a937e5eee3f389ae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732148886281 2024-11-21T00:28:08,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a9919f05f846a3b15886ceaa791e46, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732148886281 2024-11-21T00:28:08,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7688dd7394a419c819bba8e5764522e, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148886315 2024-11-21T00:28:08,232 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ded48898594a94b54de3da4ea615a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148886315 2024-11-21T00:28:08,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d93621476648ce82a54e400600b092, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732148886514 2024-11-21T00:28:08,232 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 46965fb3fc1d4c9c8b17c2ff568c4823, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732148886514 2024-11-21T00:28:08,233 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c7dc17732643bda2d4aa01351a50a7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148886892 2024-11-21T00:28:08,233 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 681991cbe39d4fa8b7881a49f4a5f7d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148886896 2024-11-21T00:28:08,249 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:08,270 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:08,271 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/d534fd29245c4e37b349f5e41b8b3465 is 50, key is test_row_0/C:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:08,277 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411216fd698fdedb44e04809f4e848cce77bc_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:08,280 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411216fd698fdedb44e04809f4e848cce77bc_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:08,281 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216fd698fdedb44e04809f4e848cce77bc_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:08,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T00:28:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,287 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T00:28:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:28:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:08,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742237_1413 (size=4469) 2024-11-21T00:28:08,307 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#344 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:08,307 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/8bd330880a7046deab132fd85213c20b is 175, key is test_row_0/A:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:08,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742236_1412 (size=12697) 2024-11-21T00:28:08,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121daf58a4f13614b348ac5865e86ce54cc_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148887617/Put/seqid=0 2024-11-21T00:28:08,321 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/d534fd29245c4e37b349f5e41b8b3465 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d534fd29245c4e37b349f5e41b8b3465 2024-11-21T00:28:08,327 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into d534fd29245c4e37b349f5e41b8b3465(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:08,327 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:08,327 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=12, startTime=1732148888229; duration=0sec 2024-11-21T00:28:08,327 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:08,327 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:08,327 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:08,328 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:08,328 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:08,329 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,329 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/82a50ab151c14cf2a79286a3cd42c8d8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/c3a185cb69d14b698a3fcd4e53db4ee2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f91dd6bce8ac4623aa0794f50c41bfdc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/e40e944025b54cc78393235de7506078] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=47.9 K 2024-11-21T00:28:08,329 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 82a50ab151c14cf2a79286a3cd42c8d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732148886281 2024-11-21T00:28:08,329 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a185cb69d14b698a3fcd4e53db4ee2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148886315 2024-11-21T00:28:08,329 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f91dd6bce8ac4623aa0794f50c41bfdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732148886514 2024-11-21T00:28:08,330 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e40e944025b54cc78393235de7506078, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148886896 2024-11-21T00:28:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148948337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148948337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742239_1415 (size=12454) 2024-11-21T00:28:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148948337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148948338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:08,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742238_1414 (size=31651) 2024-11-21T00:28:08,365 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121daf58a4f13614b348ac5865e86ce54cc_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121daf58a4f13614b348ac5865e86ce54cc_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ed52eb9432d945a88c78550484d5e50a, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ed52eb9432d945a88c78550484d5e50a is 175, key is test_row_0/A:col10/1732148887617/Put/seqid=0 2024-11-21T00:28:08,374 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:08,375 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5614d4d828ef4bf7a2b1d4742e9340ac is 50, key is test_row_0/B:col10/1732148886906/Put/seqid=0 2024-11-21T00:28:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148948360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742240_1416 (size=31255) 2024-11-21T00:28:08,393 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ed52eb9432d945a88c78550484d5e50a 2024-11-21T00:28:08,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742241_1417 (size=12697) 2024-11-21T00:28:08,400 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5614d4d828ef4bf7a2b1d4742e9340ac as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5614d4d828ef4bf7a2b1d4742e9340ac 2024-11-21T00:28:08,407 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into 5614d4d828ef4bf7a2b1d4742e9340ac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:08,407 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:08,407 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=12, startTime=1732148888229; duration=0sec 2024-11-21T00:28:08,408 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:08,408 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8584621b45c1464eb28c10fd8aab79ee is 50, key is test_row_0/B:col10/1732148887617/Put/seqid=0 2024-11-21T00:28:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742242_1418 (size=12301) 2024-11-21T00:28:08,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148948472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148948472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148948472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148948476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148948482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148948692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148948696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148948702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148948704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148948704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:08,769 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/8bd330880a7046deab132fd85213c20b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/8bd330880a7046deab132fd85213c20b 2024-11-21T00:28:08,774 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into 8bd330880a7046deab132fd85213c20b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:08,774 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:08,774 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=12, startTime=1732148888229; duration=0sec 2024-11-21T00:28:08,774 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:08,774 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:28:08,834 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8584621b45c1464eb28c10fd8aab79ee 2024-11-21T00:28:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/a42a75066964478dbae35812e94692ba is 50, key is test_row_0/C:col10/1732148887617/Put/seqid=0 2024-11-21T00:28:08,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742243_1419 (size=12301) 2024-11-21T00:28:08,874 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/a42a75066964478dbae35812e94692ba 2024-11-21T00:28:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ed52eb9432d945a88c78550484d5e50a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a 2024-11-21T00:28:08,882 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a, entries=150, sequenceid=271, filesize=30.5 K 2024-11-21T00:28:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8584621b45c1464eb28c10fd8aab79ee as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8584621b45c1464eb28c10fd8aab79ee 2024-11-21T00:28:08,887 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8584621b45c1464eb28c10fd8aab79ee, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:28:08,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/a42a75066964478dbae35812e94692ba as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a42a75066964478dbae35812e94692ba 2024-11-21T00:28:08,893 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a42a75066964478dbae35812e94692ba, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T00:28:08,895 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5776037b580ebf80c0ab7fa6d7225dd4 in 608ms, sequenceid=271, compaction requested=false 2024-11-21T00:28:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T00:28:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T00:28:08,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T00:28:08,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2320 sec 2024-11-21T00:28:08,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.2500 sec 2024-11-21T00:28:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:09,017 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:09,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T00:28:09,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a5538070be71467389e35957f74d2995_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:09,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742244_1420 (size=14994) 2024-11-21T00:28:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148949072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148949077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148949079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148949079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148949079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148949192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148949202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148949203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148949203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148949203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148949402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148949422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148949422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148949423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148949423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:09,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a5538070be71467389e35957f74d2995_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a5538070be71467389e35957f74d2995_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:09,467 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/db19eb80a5d54677aa798c29e581c369, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:09,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/db19eb80a5d54677aa798c29e581c369 is 175, key is test_row_0/A:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742245_1421 (size=39949) 2024-11-21T00:28:09,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/db19eb80a5d54677aa798c29e581c369 2024-11-21T00:28:09,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/bb9c007cd9bb4acca37916c22865af81 is 50, key is test_row_0/B:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:09,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742246_1422 (size=12301) 2024-11-21T00:28:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148949721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148949732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148949732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148949745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148949745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T00:28:09,822 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:09,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-21T00:28:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:28:09,827 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:09,827 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:09,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:28:09,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/bb9c007cd9bb4acca37916c22865af81 2024-11-21T00:28:09,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:28:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:09,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:09,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e66a99cdc54a4c6ca49d0e1b0be95591 is 50, key is test_row_0/C:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742247_1423 (size=12301) 2024-11-21T00:28:10,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e66a99cdc54a4c6ca49d0e1b0be95591 2024-11-21T00:28:10,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/db19eb80a5d54677aa798c29e581c369 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369 2024-11-21T00:28:10,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369, entries=200, sequenceid=290, filesize=39.0 K 2024-11-21T00:28:10,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/bb9c007cd9bb4acca37916c22865af81 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/bb9c007cd9bb4acca37916c22865af81 2024-11-21T00:28:10,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/bb9c007cd9bb4acca37916c22865af81, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:28:10,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e66a99cdc54a4c6ca49d0e1b0be95591 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e66a99cdc54a4c6ca49d0e1b0be95591 2024-11-21T00:28:10,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e66a99cdc54a4c6ca49d0e1b0be95591, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:28:10,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5776037b580ebf80c0ab7fa6d7225dd4 in 1012ms, sequenceid=290, compaction requested=true 2024-11-21T00:28:10,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:10,030 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:10,030 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:10,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:10,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:10,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:10,031 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:10,031 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/8bd330880a7046deab132fd85213c20b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=100.4 K 2024-11-21T00:28:10,031 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:10,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/8bd330880a7046deab132fd85213c20b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369] 2024-11-21T00:28:10,031 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd330880a7046deab132fd85213c20b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148886896 2024-11-21T00:28:10,031 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:10,031 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:10,032 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:10,032 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5614d4d828ef4bf7a2b1d4742e9340ac, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8584621b45c1464eb28c10fd8aab79ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/bb9c007cd9bb4acca37916c22865af81] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=36.4 K 2024-11-21T00:28:10,032 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed52eb9432d945a88c78550484d5e50a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148887617 2024-11-21T00:28:10,033 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5614d4d828ef4bf7a2b1d4742e9340ac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148886896 2024-11-21T00:28:10,034 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting db19eb80a5d54677aa798c29e581c369, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148888322 2024-11-21T00:28:10,035 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8584621b45c1464eb28c10fd8aab79ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148887617 2024-11-21T00:28:10,035 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bb9c007cd9bb4acca37916c22865af81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148888322 2024-11-21T00:28:10,046 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:10,048 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112109376c437dc24484ac4b347b03237370_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:10,049 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:10,050 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8168cbae684e4d0cbbf14f92001804ed is 50, key is test_row_0/B:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:10,050 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112109376c437dc24484ac4b347b03237370_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:10,050 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112109376c437dc24484ac4b347b03237370_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:10,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742248_1424 (size=12949) 2024-11-21T00:28:10,121 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/8168cbae684e4d0cbbf14f92001804ed as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8168cbae684e4d0cbbf14f92001804ed 2024-11-21T00:28:10,129 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into 8168cbae684e4d0cbbf14f92001804ed(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:10,129 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:10,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742249_1425 (size=4469) 2024-11-21T00:28:10,130 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=13, startTime=1732148890030; duration=0sec 2024-11-21T00:28:10,130 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:10,130 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:10,130 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:10,131 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:10,131 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:10,131 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:10,132 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d534fd29245c4e37b349f5e41b8b3465, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a42a75066964478dbae35812e94692ba, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e66a99cdc54a4c6ca49d0e1b0be95591] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=36.4 K 2024-11-21T00:28:10,132 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d534fd29245c4e37b349f5e41b8b3465, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148886896 2024-11-21T00:28:10,132 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a42a75066964478dbae35812e94692ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732148887617 2024-11-21T00:28:10,133 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e66a99cdc54a4c6ca49d0e1b0be95591, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148888322 2024-11-21T00:28:10,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T00:28:10,139 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#353 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:10,140 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/0913e150365b48548c5ac302f19eeb62 is 175, key is test_row_0/A:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:10,140 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T00:28:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:28:10,144 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:10,145 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/0d8a2215ad464ee7b8663a3bbc099cd3 is 50, key is test_row_0/C:col10/1732148889016/Put/seqid=0 2024-11-21T00:28:10,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742250_1426 (size=31903) 2024-11-21T00:28:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a6124f8d136d48448cc927866894afd8_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148889071/Put/seqid=0 2024-11-21T00:28:10,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742251_1427 (size=12949) 2024-11-21T00:28:10,237 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/0d8a2215ad464ee7b8663a3bbc099cd3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/0d8a2215ad464ee7b8663a3bbc099cd3 2024-11-21T00:28:10,243 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into 0d8a2215ad464ee7b8663a3bbc099cd3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:10,243 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:10,243 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=13, startTime=1732148890030; duration=0sec 2024-11-21T00:28:10,243 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:10,243 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:10,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742252_1428 (size=12454) 2024-11-21T00:28:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:10,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:10,265 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a6124f8d136d48448cc927866894afd8_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6124f8d136d48448cc927866894afd8_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/c05851418ce345acbec6c1921288a81a, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:10,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/c05851418ce345acbec6c1921288a81a is 175, key is test_row_0/A:col10/1732148889071/Put/seqid=0 2024-11-21T00:28:10,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742253_1429 (size=31255) 2024-11-21T00:28:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148950303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148950306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,325 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/c05851418ce345acbec6c1921288a81a 2024-11-21T00:28:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148950311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148950325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148950329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/455836cd58084e57ac0663f32431b7d7 is 50, key is test_row_0/B:col10/1732148889071/Put/seqid=0 2024-11-21T00:28:10,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742254_1430 (size=12301) 2024-11-21T00:28:10,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148950433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148950433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148950433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:28:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148950456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148950462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,614 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/0913e150365b48548c5ac302f19eeb62 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/0913e150365b48548c5ac302f19eeb62 2024-11-21T00:28:10,621 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into 0913e150365b48548c5ac302f19eeb62(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:10,621 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:10,621 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=13, startTime=1732148890030; duration=0sec 2024-11-21T00:28:10,621 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:10,621 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:10,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148950653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148950653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148950654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148950683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148950683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:10,798 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/455836cd58084e57ac0663f32431b7d7 2024-11-21T00:28:10,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b9ade6934a894d01b767d991f1d049c4 is 50, key is test_row_0/C:col10/1732148889071/Put/seqid=0 2024-11-21T00:28:10,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742255_1431 (size=12301) 2024-11-21T00:28:10,821 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b9ade6934a894d01b767d991f1d049c4 2024-11-21T00:28:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/c05851418ce345acbec6c1921288a81a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a 2024-11-21T00:28:10,833 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a, entries=150, sequenceid=311, filesize=30.5 K 2024-11-21T00:28:10,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/455836cd58084e57ac0663f32431b7d7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/455836cd58084e57ac0663f32431b7d7 2024-11-21T00:28:10,857 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/455836cd58084e57ac0663f32431b7d7, entries=150, sequenceid=311, filesize=12.0 K 2024-11-21T00:28:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b9ade6934a894d01b767d991f1d049c4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b9ade6934a894d01b767d991f1d049c4 2024-11-21T00:28:10,876 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b9ade6934a894d01b767d991f1d049c4, entries=150, sequenceid=311, filesize=12.0 K 2024-11-21T00:28:10,877 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 5776037b580ebf80c0ab7fa6d7225dd4 in 737ms, sequenceid=311, compaction requested=false 2024-11-21T00:28:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-21T00:28:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-21T00:28:10,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T00:28:10,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0520 sec 2024-11-21T00:28:10,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0590 sec 2024-11-21T00:28:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T00:28:10,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:10,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T00:28:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:10,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:10,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:10,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:10,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:10,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:28:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:11,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112115dd2b90d91647e2a19dee2812fc99e2_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148951010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148951012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148951023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148951024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742256_1432 (size=12454) 2024-11-21T00:28:11,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148951028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,045 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112115dd2b90d91647e2a19dee2812fc99e2_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112115dd2b90d91647e2a19dee2812fc99e2_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:11,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/6ce8d4a4e6e64d47a2ff50bf390f35d3, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:11,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/6ce8d4a4e6e64d47a2ff50bf390f35d3 is 175, key is test_row_0/A:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:11,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742257_1433 (size=31255) 2024-11-21T00:28:11,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:11,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148951133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148951133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148951142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148951142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148951152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:11,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148951352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148951352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148951362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148951362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148951372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:11,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,487 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/6ce8d4a4e6e64d47a2ff50bf390f35d3 2024-11-21T00:28:11,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5646a28171564e488822226b79e14e29 is 50, key is test_row_0/B:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742258_1434 (size=12301) 2024-11-21T00:28:11,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5646a28171564e488822226b79e14e29 2024-11-21T00:28:11,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/d949330cd18f49b095f4e1a8c029de4e is 50, key is test_row_0/C:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742259_1435 (size=12301) 2024-11-21T00:28:11,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/d949330cd18f49b095f4e1a8c029de4e 2024-11-21T00:28:11,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:11,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:11,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/6ce8d4a4e6e64d47a2ff50bf390f35d3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3 2024-11-21T00:28:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3, entries=150, sequenceid=331, filesize=30.5 K 2024-11-21T00:28:11,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5646a28171564e488822226b79e14e29 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5646a28171564e488822226b79e14e29 2024-11-21T00:28:11,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5646a28171564e488822226b79e14e29, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T00:28:11,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/d949330cd18f49b095f4e1a8c029de4e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d949330cd18f49b095f4e1a8c029de4e 2024-11-21T00:28:11,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d949330cd18f49b095f4e1a8c029de4e, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T00:28:11,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5776037b580ebf80c0ab7fa6d7225dd4 in 663ms, sequenceid=331, compaction requested=true 2024-11-21T00:28:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:11,631 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:11,632 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:11,632 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:11,633 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,633 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/0913e150365b48548c5ac302f19eeb62, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=92.2 K 2024-11-21T00:28:11,633 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,633 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/0913e150365b48548c5ac302f19eeb62, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3] 2024-11-21T00:28:11,633 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0913e150365b48548c5ac302f19eeb62, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148888322 2024-11-21T00:28:11,634 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c05851418ce345acbec6c1921288a81a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148889071 2024-11-21T00:28:11,634 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce8d4a4e6e64d47a2ff50bf390f35d3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148890292 2024-11-21T00:28:11,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:11,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:11,638 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:11,639 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:11,639 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:11,639 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,639 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8168cbae684e4d0cbbf14f92001804ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/455836cd58084e57ac0663f32431b7d7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5646a28171564e488822226b79e14e29] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=36.7 K 2024-11-21T00:28:11,640 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8168cbae684e4d0cbbf14f92001804ed, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148888322 2024-11-21T00:28:11,640 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455836cd58084e57ac0663f32431b7d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148889071 2024-11-21T00:28:11,641 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:11,641 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5646a28171564e488822226b79e14e29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148890292 2024-11-21T00:28:11,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:11,649 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411212911bd426bb148a9892968c354656d57_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:11,651 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411212911bd426bb148a9892968c354656d57_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:11,651 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212911bd426bb148a9892968c354656d57_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:11,654 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:11,654 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5a8cec96fa154295a0fc3de7cdeb1309 is 50, key is test_row_0/B:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:11,668 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:11,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:11,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742260_1436 (size=4469) 2024-11-21T00:28:11,698 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#362 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:11,698 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d20d982765aa49a5a2acabc099947a60 is 175, key is test_row_0/A:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742261_1437 (size=13051) 2024-11-21T00:28:11,709 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/5a8cec96fa154295a0fc3de7cdeb1309 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5a8cec96fa154295a0fc3de7cdeb1309 2024-11-21T00:28:11,714 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into 5a8cec96fa154295a0fc3de7cdeb1309(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:11,714 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:11,714 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=13, startTime=1732148891638; duration=0sec 2024-11-21T00:28:11,714 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:11,714 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:11,715 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:11,717 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:11,717 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:11,717 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,717 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/0d8a2215ad464ee7b8663a3bbc099cd3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b9ade6934a894d01b767d991f1d049c4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d949330cd18f49b095f4e1a8c029de4e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=36.7 K 2024-11-21T00:28:11,719 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d8a2215ad464ee7b8663a3bbc099cd3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148888322 2024-11-21T00:28:11,720 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ade6934a894d01b767d991f1d049c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732148889071 2024-11-21T00:28:11,720 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d949330cd18f49b095f4e1a8c029de4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148890292 2024-11-21T00:28:11,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216f55643bb931496e93b516eebeb2e35a_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148891667/Put/seqid=0 2024-11-21T00:28:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148951721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148951724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148951724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,742 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,742 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/32b5a0fe2fd74a0c871e19f91be0d175 is 50, key is test_row_0/C:col10/1732148890292/Put/seqid=0 2024-11-21T00:28:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742262_1438 (size=32005) 2024-11-21T00:28:11,756 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/d20d982765aa49a5a2acabc099947a60 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d20d982765aa49a5a2acabc099947a60 2024-11-21T00:28:11,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148951742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148951743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742263_1439 (size=14994) 2024-11-21T00:28:11,771 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:11,776 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into d20d982765aa49a5a2acabc099947a60(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:11,776 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:11,776 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=13, startTime=1732148891631; duration=0sec 2024-11-21T00:28:11,776 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:11,776 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:11,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216f55643bb931496e93b516eebeb2e35a_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216f55643bb931496e93b516eebeb2e35a_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:11,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/98fc6cc9a33b48bf99a6f0c778a515c0, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:11,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/98fc6cc9a33b48bf99a6f0c778a515c0 is 175, key is test_row_0/A:col10/1732148891667/Put/seqid=0 2024-11-21T00:28:11,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742264_1440 (size=13051) 2024-11-21T00:28:11,819 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/32b5a0fe2fd74a0c871e19f91be0d175 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/32b5a0fe2fd74a0c871e19f91be0d175 2024-11-21T00:28:11,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742265_1441 (size=39949) 2024-11-21T00:28:11,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/98fc6cc9a33b48bf99a6f0c778a515c0 2024-11-21T00:28:11,839 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into 32b5a0fe2fd74a0c871e19f91be0d175(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:11,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:11,839 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=13, startTime=1732148891646; duration=0sec 2024-11-21T00:28:11,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:11,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:11,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/36bb5876121e4c56a9ad39bc80ef23a6 is 50, key is test_row_0/B:col10/1732148891667/Put/seqid=0 2024-11-21T00:28:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148951852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148951852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148951852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742266_1442 (size=12301) 2024-11-21T00:28:11,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/36bb5876121e4c56a9ad39bc80ef23a6 2024-11-21T00:28:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148951861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/59d25f4c40414204aff28dfdbf03bde9 is 50, key is test_row_0/C:col10/1732148891667/Put/seqid=0 2024-11-21T00:28:11,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148951872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:11,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742267_1443 (size=12301) 2024-11-21T00:28:11,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:11,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:12,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148952062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148952062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148952062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:12,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148952083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148952092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:12,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:12,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:12,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:12,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:12,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:12,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/59d25f4c40414204aff28dfdbf03bde9 2024-11-21T00:28:12,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/98fc6cc9a33b48bf99a6f0c778a515c0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0 2024-11-21T00:28:12,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0, entries=200, sequenceid=350, filesize=39.0 K 2024-11-21T00:28:12,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/36bb5876121e4c56a9ad39bc80ef23a6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/36bb5876121e4c56a9ad39bc80ef23a6 2024-11-21T00:28:12,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/36bb5876121e4c56a9ad39bc80ef23a6, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T00:28:12,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/59d25f4c40414204aff28dfdbf03bde9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/59d25f4c40414204aff28dfdbf03bde9 2024-11-21T00:28:12,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/59d25f4c40414204aff28dfdbf03bde9, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T00:28:12,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5776037b580ebf80c0ab7fa6d7225dd4 in 683ms, sequenceid=350, compaction requested=false 2024-11-21T00:28:12,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:12,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T00:28:12,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:12,389 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:28:12,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. as already flushing 2024-11-21T00:28:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:12,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112140688e09af9c4cfca04a8a84d888f134_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148891721/Put/seqid=0 2024-11-21T00:28:12,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742268_1444 (size=12454) 2024-11-21T00:28:12,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148952440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148952445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148952445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148952452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148952458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148952572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148952572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148952572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148952573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148952582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148952793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148952793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148952793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148952793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148952802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:12,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:12,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:12,837 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112140688e09af9c4cfca04a8a84d888f134_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112140688e09af9c4cfca04a8a84d888f134_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ecf5f22b6d094284b5f8b36509cfbb0f, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ecf5f22b6d094284b5f8b36509cfbb0f is 175, key is test_row_0/A:col10/1732148891721/Put/seqid=0 2024-11-21T00:28:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742269_1445 (size=31255) 2024-11-21T00:28:12,854 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ecf5f22b6d094284b5f8b36509cfbb0f 2024-11-21T00:28:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/1ab2b8932b514f0c9a683a0b236b6e57 is 50, key is test_row_0/B:col10/1732148891721/Put/seqid=0 2024-11-21T00:28:12,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742270_1446 (size=12301) 2024-11-21T00:28:13,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148953102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148953112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148953112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148953113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148953113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,303 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/1ab2b8932b514f0c9a683a0b236b6e57 2024-11-21T00:28:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5339bf5692254dba8e2caba233f39208 is 50, key is test_row_0/C:col10/1732148891721/Put/seqid=0 2024-11-21T00:28:13,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742271_1447 (size=12301) 2024-11-21T00:28:13,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148953622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148953622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148953633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148953633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:13,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148953633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:14,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148954652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:14,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148954652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:14,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148954652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:14,661 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 5776037b580ebf80c0ab7fa6d7225dd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:14,661 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 5776037b580ebf80c0ab7fa6d7225dd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:14,662 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 5776037b580ebf80c0ab7fa6d7225dd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148954663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:14,671 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 5776037b580ebf80c0ab7fa6d7225dd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:14,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148954663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:14,672 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 5776037b580ebf80c0ab7fa6d7225dd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:16,007 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:28:16,454 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5339bf5692254dba8e2caba233f39208 2024-11-21T00:28:16,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/ecf5f22b6d094284b5f8b36509cfbb0f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f 2024-11-21T00:28:16,494 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f, entries=150, sequenceid=371, filesize=30.5 K 2024-11-21T00:28:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/1ab2b8932b514f0c9a683a0b236b6e57 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1ab2b8932b514f0c9a683a0b236b6e57 2024-11-21T00:28:16,506 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1ab2b8932b514f0c9a683a0b236b6e57, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T00:28:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5339bf5692254dba8e2caba233f39208 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5339bf5692254dba8e2caba233f39208 2024-11-21T00:28:16,511 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5339bf5692254dba8e2caba233f39208, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T00:28:16,513 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5776037b580ebf80c0ab7fa6d7225dd4 in 4124ms, sequenceid=371, compaction requested=true 2024-11-21T00:28:16,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T00:28:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T00:28:16,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T00:28:16,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.5540 sec 2024-11-21T00:28:16,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 5.5640 sec 2024-11-21T00:28:16,688 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:16,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:16,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112132bb6c446b924ef1a1225bbef3596306_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:16,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742272_1448 (size=14994) 2024-11-21T00:28:16,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:16,718 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112132bb6c446b924ef1a1225bbef3596306_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112132bb6c446b924ef1a1225bbef3596306_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:16,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3b23a755a03a49fd98d425bb043c81fd, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:16,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3b23a755a03a49fd98d425bb043c81fd is 175, key is test_row_0/A:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148956716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148956728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148956728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148956730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148956732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742273_1449 (size=39949) 2024-11-21T00:28:16,773 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3b23a755a03a49fd98d425bb043c81fd 2024-11-21T00:28:16,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/4cc959bebaa24abbb56fe730489faa38 is 50, key is test_row_0/B:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:16,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742274_1450 (size=12301) 2024-11-21T00:28:16,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/4cc959bebaa24abbb56fe730489faa38 2024-11-21T00:28:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148956842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148956848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148956853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148956853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148956853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:16,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/c097bef542f441cd9678173e06fdddcc is 50, key is test_row_0/C:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:16,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742275_1451 (size=12301) 2024-11-21T00:28:16,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/c097bef542f441cd9678173e06fdddcc 2024-11-21T00:28:16,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/3b23a755a03a49fd98d425bb043c81fd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd 2024-11-21T00:28:16,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd, entries=200, sequenceid=390, filesize=39.0 K 2024-11-21T00:28:16,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/4cc959bebaa24abbb56fe730489faa38 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/4cc959bebaa24abbb56fe730489faa38 2024-11-21T00:28:16,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/4cc959bebaa24abbb56fe730489faa38, entries=150, sequenceid=390, filesize=12.0 K 2024-11-21T00:28:16,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/c097bef542f441cd9678173e06fdddcc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/c097bef542f441cd9678173e06fdddcc 2024-11-21T00:28:16,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/c097bef542f441cd9678173e06fdddcc, entries=150, sequenceid=390, filesize=12.0 K 2024-11-21T00:28:16,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5776037b580ebf80c0ab7fa6d7225dd4 in 261ms, sequenceid=390, compaction requested=true 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:16,949 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:16,949 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:16,968 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:16,969 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:16,969 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:16,969 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d20d982765aa49a5a2acabc099947a60, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=139.8 K 2024-11-21T00:28:16,969 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:16,969 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d20d982765aa49a5a2acabc099947a60, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd] 2024-11-21T00:28:16,971 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:16,971 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:16,971 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:16,971 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5a8cec96fa154295a0fc3de7cdeb1309, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/36bb5876121e4c56a9ad39bc80ef23a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1ab2b8932b514f0c9a683a0b236b6e57, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/4cc959bebaa24abbb56fe730489faa38] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=48.8 K 2024-11-21T00:28:16,972 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20d982765aa49a5a2acabc099947a60, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148890292 2024-11-21T00:28:16,972 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a8cec96fa154295a0fc3de7cdeb1309, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148890292 2024-11-21T00:28:16,974 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98fc6cc9a33b48bf99a6f0c778a515c0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148891005 2024-11-21T00:28:16,975 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bb5876121e4c56a9ad39bc80ef23a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148891005 2024-11-21T00:28:16,977 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf5f22b6d094284b5f8b36509cfbb0f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732148891716 2024-11-21T00:28:16,977 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab2b8932b514f0c9a683a0b236b6e57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732148891716 2024-11-21T00:28:16,977 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b23a755a03a49fd98d425bb043c81fd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732148892400 2024-11-21T00:28:16,977 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc959bebaa24abbb56fe730489faa38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732148892400 2024-11-21T00:28:16,989 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:16,991 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:16,991 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/2ca7b68b50244e389dfe2753c9c44384 is 50, key is test_row_0/B:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:17,002 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211aa7ce41d1ee43eb8af7ad3716bdce9e_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:17,005 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211aa7ce41d1ee43eb8af7ad3716bdce9e_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:17,005 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211aa7ce41d1ee43eb8af7ad3716bdce9e_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:17,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742276_1452 (size=13187) 2024-11-21T00:28:17,017 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/2ca7b68b50244e389dfe2753c9c44384 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/2ca7b68b50244e389dfe2753c9c44384 2024-11-21T00:28:17,025 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into 2ca7b68b50244e389dfe2753c9c44384(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:17,025 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:17,025 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=12, startTime=1732148896949; duration=0sec 2024-11-21T00:28:17,025 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:17,025 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:17,026 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:17,027 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:17,027 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:17,027 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:17,027 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/32b5a0fe2fd74a0c871e19f91be0d175, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/59d25f4c40414204aff28dfdbf03bde9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5339bf5692254dba8e2caba233f39208, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/c097bef542f441cd9678173e06fdddcc] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=48.8 K 2024-11-21T00:28:17,027 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 32b5a0fe2fd74a0c871e19f91be0d175, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732148890292 2024-11-21T00:28:17,027 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d25f4c40414204aff28dfdbf03bde9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732148891005 2024-11-21T00:28:17,028 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5339bf5692254dba8e2caba233f39208, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732148891716 2024-11-21T00:28:17,030 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c097bef542f441cd9678173e06fdddcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732148892400 2024-11-21T00:28:17,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742277_1453 (size=4469) 2024-11-21T00:28:17,046 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:17,047 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5e38bfb5495b4ad4801c51f2c84e5096 is 50, key is test_row_0/C:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:17,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:17,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:28:17,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742278_1454 (size=13187) 2024-11-21T00:28:17,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112194b0ed9e096c4910b1af781958003410_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148896729/Put/seqid=0 2024-11-21T00:28:17,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742279_1455 (size=14994) 2024-11-21T00:28:17,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:17,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112194b0ed9e096c4910b1af781958003410_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112194b0ed9e096c4910b1af781958003410_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:17,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/1ae8f16adc5149c68b3997748b17d6e9, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:17,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/1ae8f16adc5149c68b3997748b17d6e9 is 175, key is test_row_0/A:col10/1732148896729/Put/seqid=0 2024-11-21T00:28:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148957117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742280_1456 (size=39949) 2024-11-21T00:28:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148957119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148957120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148957121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,144 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/1ae8f16adc5149c68b3997748b17d6e9 2024-11-21T00:28:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148957133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f1f378114dc644f0ae96df881cf2c2c1 is 50, key is test_row_0/B:col10/1732148896729/Put/seqid=0 2024-11-21T00:28:17,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742281_1457 (size=12301) 2024-11-21T00:28:17,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148957242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148957242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148957252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148957252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148957252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,434 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#375 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:17,435 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/96b8d3e1d5f741a09dc3700db732a7ee is 175, key is test_row_0/A:col10/1732148892400/Put/seqid=0 2024-11-21T00:28:17,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148957453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148957453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742282_1458 (size=32141) 2024-11-21T00:28:17,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148957473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148957474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148957474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,499 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/96b8d3e1d5f741a09dc3700db732a7ee as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96b8d3e1d5f741a09dc3700db732a7ee 2024-11-21T00:28:17,522 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into 96b8d3e1d5f741a09dc3700db732a7ee(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:17,522 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:17,522 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=12, startTime=1732148896949; duration=0sec 2024-11-21T00:28:17,523 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:17,523 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:17,527 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/5e38bfb5495b4ad4801c51f2c84e5096 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5e38bfb5495b4ad4801c51f2c84e5096 2024-11-21T00:28:17,534 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into 5e38bfb5495b4ad4801c51f2c84e5096(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:17,534 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:17,534 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=12, startTime=1732148896949; duration=0sec 2024-11-21T00:28:17,534 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:17,534 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:17,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f1f378114dc644f0ae96df881cf2c2c1 2024-11-21T00:28:17,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e4963007fcee4d15a28a3f547c344158 is 50, key is test_row_0/C:col10/1732148896729/Put/seqid=0 2024-11-21T00:28:17,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742283_1459 (size=12301) 2024-11-21T00:28:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148957772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148957772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148957792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148957792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148957802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e4963007fcee4d15a28a3f547c344158 2024-11-21T00:28:18,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/1ae8f16adc5149c68b3997748b17d6e9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9 2024-11-21T00:28:18,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9, entries=200, sequenceid=409, filesize=39.0 K 2024-11-21T00:28:18,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f1f378114dc644f0ae96df881cf2c2c1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f1f378114dc644f0ae96df881cf2c2c1 2024-11-21T00:28:18,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f1f378114dc644f0ae96df881cf2c2c1, entries=150, sequenceid=409, filesize=12.0 K 2024-11-21T00:28:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/e4963007fcee4d15a28a3f547c344158 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e4963007fcee4d15a28a3f547c344158 2024-11-21T00:28:18,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e4963007fcee4d15a28a3f547c344158, entries=150, sequenceid=409, filesize=12.0 K 2024-11-21T00:28:18,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5776037b580ebf80c0ab7fa6d7225dd4 in 994ms, sequenceid=409, compaction requested=false 2024-11-21T00:28:18,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:18,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:18,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a72b2b4f210a4c4cb679350df0fb6dd8_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:18,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742284_1460 (size=14994) 2024-11-21T00:28:18,317 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:18,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a72b2b4f210a4c4cb679350df0fb6dd8_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a72b2b4f210a4c4cb679350df0fb6dd8_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:18,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/702ea2d63f4348bc95dc18b9aafb7d2e, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:18,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/702ea2d63f4348bc95dc18b9aafb7d2e is 175, key is test_row_0/A:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:18,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742285_1461 (size=39949) 2024-11-21T00:28:18,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/702ea2d63f4348bc95dc18b9aafb7d2e 2024-11-21T00:28:18,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148958326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148958329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148958330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/29c3fab8ccb3462abc0eaee88cb78280 is 50, key is test_row_0/B:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:18,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148958345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148958345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742286_1462 (size=12301) 2024-11-21T00:28:18,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/29c3fab8ccb3462abc0eaee88cb78280 2024-11-21T00:28:18,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/69933788998f47798b31f2e47a74f0fa is 50, key is test_row_0/C:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:18,404 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-21T00:28:18,404 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,405 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,405 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,405 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-21T00:28:18,405 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,405 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,405 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,413 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-21T00:28:18,413 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,413 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,413 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,414 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-11-21T00:28:18,414 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,414 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,414 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,418 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-21T00:28:18,418 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,418 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,418 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742287_1463 (size=12301) 2024-11-21T00:28:18,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148958452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148958452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148958453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148958462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148958462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56530 deadline: 1732148958662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1732148958663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1732148958663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56504 deadline: 1732148958672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56462 deadline: 1732148958672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:18,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/69933788998f47798b31f2e47a74f0fa 2024-11-21T00:28:18,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/702ea2d63f4348bc95dc18b9aafb7d2e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e 2024-11-21T00:28:18,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e, entries=200, sequenceid=430, filesize=39.0 K 2024-11-21T00:28:18,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/29c3fab8ccb3462abc0eaee88cb78280 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/29c3fab8ccb3462abc0eaee88cb78280 2024-11-21T00:28:18,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/29c3fab8ccb3462abc0eaee88cb78280, entries=150, sequenceid=430, filesize=12.0 K 2024-11-21T00:28:18,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/69933788998f47798b31f2e47a74f0fa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/69933788998f47798b31f2e47a74f0fa 2024-11-21T00:28:18,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/69933788998f47798b31f2e47a74f0fa, entries=150, sequenceid=430, filesize=12.0 K 2024-11-21T00:28:18,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5776037b580ebf80c0ab7fa6d7225dd4 in 541ms, sequenceid=430, compaction requested=true 2024-11-21T00:28:18,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:18,835 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:18,836 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5776037b580ebf80c0ab7fa6d7225dd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/A is initiating minor compaction (all files) 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/B is initiating minor compaction (all files) 2024-11-21T00:28:18,839 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/A in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:18,839 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/B in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:18,839 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96b8d3e1d5f741a09dc3700db732a7ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=109.4 K 2024-11-21T00:28:18,839 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/2ca7b68b50244e389dfe2753c9c44384, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f1f378114dc644f0ae96df881cf2c2c1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/29c3fab8ccb3462abc0eaee88cb78280] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=36.9 K 2024-11-21T00:28:18,839 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96b8d3e1d5f741a09dc3700db732a7ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e] 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca7b68b50244e389dfe2753c9c44384, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732148892400 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b8d3e1d5f741a09dc3700db732a7ee, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732148892400 2024-11-21T00:28:18,839 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f378114dc644f0ae96df881cf2c2c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732148896721 2024-11-21T00:28:18,840 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae8f16adc5149c68b3997748b17d6e9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732148896721 2024-11-21T00:28:18,840 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c3fab8ccb3462abc0eaee88cb78280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732148897117 2024-11-21T00:28:18,840 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702ea2d63f4348bc95dc18b9aafb7d2e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732148897117 2024-11-21T00:28:18,851 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:18,858 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ddbe315df4344f2f84515bcd88687590_5776037b580ebf80c0ab7fa6d7225dd4 store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:18,859 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#B#compaction#383 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:18,859 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/d701681e1f1842eaabc98be1b5255d24 is 50, key is test_row_0/B:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:18,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742288_1464 (size=13289) 2024-11-21T00:28:18,909 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ddbe315df4344f2f84515bcd88687590_5776037b580ebf80c0ab7fa6d7225dd4, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:18,910 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ddbe315df4344f2f84515bcd88687590_5776037b580ebf80c0ab7fa6d7225dd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:18,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742289_1465 (size=4469) 2024-11-21T00:28:18,950 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#A#compaction#384 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:18,950 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/dd0c158f975e48d7b6191ea813bbccdf is 175, key is test_row_0/A:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:18,990 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-21T00:28:18,990 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,990 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,990 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:18,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T00:28:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:18,998 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-21T00:28:18,998 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,998 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,998 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:18,999 INFO [Thread-1587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1587. 2024-11-21T00:28:18,999 DEBUG [Thread-1587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:18,999 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:18,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742290_1466 (size=32243) 2024-11-21T00:28:18,999 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,002 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-21T00:28:19,002 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:19,003 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,003 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:19,007 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-21T00:28:19,007 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:19,007 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,007 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:19,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:19,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112151028109d32242129b79b55de9e53bd8_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_0/A:col10/1732148898327/Put/seqid=0 2024-11-21T00:28:19,065 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/dd0c158f975e48d7b6191ea813bbccdf as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/dd0c158f975e48d7b6191ea813bbccdf 2024-11-21T00:28:19,078 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/A of 5776037b580ebf80c0ab7fa6d7225dd4 into dd0c158f975e48d7b6191ea813bbccdf(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:19,078 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:19,078 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/A, priority=13, startTime=1732148898835; duration=0sec 2024-11-21T00:28:19,078 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:19,078 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:A 2024-11-21T00:28:19,079 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:19,080 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:19,080 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5776037b580ebf80c0ab7fa6d7225dd4/C is initiating minor compaction (all files) 2024-11-21T00:28:19,080 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5776037b580ebf80c0ab7fa6d7225dd4/C in TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:19,080 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5e38bfb5495b4ad4801c51f2c84e5096, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e4963007fcee4d15a28a3f547c344158, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/69933788998f47798b31f2e47a74f0fa] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp, totalSize=36.9 K 2024-11-21T00:28:19,080 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e38bfb5495b4ad4801c51f2c84e5096, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732148892400 2024-11-21T00:28:19,081 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4963007fcee4d15a28a3f547c344158, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732148896721 2024-11-21T00:28:19,081 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69933788998f47798b31f2e47a74f0fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732148897117 2024-11-21T00:28:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742291_1467 (size=12454) 2024-11-21T00:28:19,119 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5776037b580ebf80c0ab7fa6d7225dd4#C#compaction#386 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:19,121 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/93ed393bbc784c8a8cca72d20f1982d7 is 50, key is test_row_0/C:col10/1732148897117/Put/seqid=0 2024-11-21T00:28:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T00:28:19,152 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:19,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:28:19,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-21T00:28:19,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T00:28:19,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 951 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2853 rows 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 969 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2901 rows 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 971 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2907 rows 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 966 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2889 rows 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 967 2024-11-21T00:28:19,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2894 rows 2024-11-21T00:28:19,153 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:28:19,153 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:28:19,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:19,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:19,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:28:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:19,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742292_1468 (size=13289) 2024-11-21T00:28:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:19,176 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/93ed393bbc784c8a8cca72d20f1982d7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/93ed393bbc784c8a8cca72d20f1982d7 2024-11-21T00:28:19,182 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/C of 5776037b580ebf80c0ab7fa6d7225dd4 into 93ed393bbc784c8a8cca72d20f1982d7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:19,182 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:19,182 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/C, priority=13, startTime=1732148898836; duration=0sec 2024-11-21T00:28:19,182 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:19,183 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:C 2024-11-21T00:28:19,183 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148899183"}]},"ts":"1732148899183"} 2024-11-21T00:28:19,193 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:28:19,193 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:28:19,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:28:19,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, UNASSIGN}] 2024-11-21T00:28:19,205 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, UNASSIGN 2024-11-21T00:28:19,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:19,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, UNASSIGN because future has completed 2024-11-21T00:28:19,208 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:28:19,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:28:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:19,307 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/d701681e1f1842eaabc98be1b5255d24 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d701681e1f1842eaabc98be1b5255d24 2024-11-21T00:28:19,318 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5776037b580ebf80c0ab7fa6d7225dd4/B of 5776037b580ebf80c0ab7fa6d7225dd4 into d701681e1f1842eaabc98be1b5255d24(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:19,318 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:19,318 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., storeName=5776037b580ebf80c0ab7fa6d7225dd4/B, priority=13, startTime=1732148898836; duration=0sec 2024-11-21T00:28:19,319 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:19,319 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5776037b580ebf80c0ab7fa6d7225dd4:B 2024-11-21T00:28:19,365 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:19,365 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:28:19,365 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 5776037b580ebf80c0ab7fa6d7225dd4, disabling compactions & flushes 2024-11-21T00:28:19,365 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:19,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:19,516 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112151028109d32242129b79b55de9e53bd8_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112151028109d32242129b79b55de9e53bd8_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:19,517 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/4a84f4f89b684ac8b9f5607f37b62a82, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:19,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/4a84f4f89b684ac8b9f5607f37b62a82 is 175, key is test_row_0/A:col10/1732148898327/Put/seqid=0 2024-11-21T00:28:19,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742293_1469 (size=31255) 2024-11-21T00:28:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:19,973 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/4a84f4f89b684ac8b9f5607f37b62a82 2024-11-21T00:28:20,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f540d3dbeea043d0bc04648625186da4 is 50, key is test_row_0/B:col10/1732148898327/Put/seqid=0 2024-11-21T00:28:20,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742294_1470 (size=12301) 2024-11-21T00:28:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:20,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f540d3dbeea043d0bc04648625186da4 2024-11-21T00:28:20,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b56fe9ea4123492583a4876815330a0e is 50, key is test_row_0/C:col10/1732148898327/Put/seqid=0 2024-11-21T00:28:20,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742295_1471 (size=12301) 2024-11-21T00:28:20,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b56fe9ea4123492583a4876815330a0e 2024-11-21T00:28:20,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/4a84f4f89b684ac8b9f5607f37b62a82 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/4a84f4f89b684ac8b9f5607f37b62a82 2024-11-21T00:28:20,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/4a84f4f89b684ac8b9f5607f37b62a82, entries=150, sequenceid=452, filesize=30.5 K 2024-11-21T00:28:20,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/f540d3dbeea043d0bc04648625186da4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f540d3dbeea043d0bc04648625186da4 2024-11-21T00:28:20,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f540d3dbeea043d0bc04648625186da4, entries=150, sequenceid=452, filesize=12.0 K 2024-11-21T00:28:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/b56fe9ea4123492583a4876815330a0e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b56fe9ea4123492583a4876815330a0e 2024-11-21T00:28:21,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b56fe9ea4123492583a4876815330a0e, entries=150, sequenceid=452, filesize=12.0 K 2024-11-21T00:28:21,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=6.71 KB/6870 for 5776037b580ebf80c0ab7fa6d7225dd4 in 2034ms, sequenceid=452, compaction requested=false 2024-11-21T00:28:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5776037b580ebf80c0ab7fa6d7225dd4: 2024-11-21T00:28:21,030 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:21,030 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:21,030 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. after waiting 0 ms 2024-11-21T00:28:21,030 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:21,030 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 5776037b580ebf80c0ab7fa6d7225dd4 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-21T00:28:21,069 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218eb6ff3f951240099ed39388e2f0c215_5776037b580ebf80c0ab7fa6d7225dd4 is 50, key is test_row_2/A:col10/1732148899002/Put/seqid=0 2024-11-21T00:28:21,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742296_1472 (size=7374) 2024-11-21T00:28:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:21,537 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:21,570 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218eb6ff3f951240099ed39388e2f0c215_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218eb6ff3f951240099ed39388e2f0c215_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:21,580 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/9536ea0b13a0427e8da04d83644b54bd, store: [table=TestAcidGuarantees family=A region=5776037b580ebf80c0ab7fa6d7225dd4] 2024-11-21T00:28:21,581 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/9536ea0b13a0427e8da04d83644b54bd is 175, key is test_row_2/A:col10/1732148899002/Put/seqid=0 2024-11-21T00:28:21,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742297_1473 (size=13865) 2024-11-21T00:28:21,607 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/36d88c536518438ea9c497b16d5ae5b2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/05e73b0a1e1148a5983936e4f27fffb0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/15a9919f05f846a3b15886ceaa791e46, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/8bd330880a7046deab132fd85213c20b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/0913e150365b48548c5ac302f19eeb62, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d20d982765aa49a5a2acabc099947a60, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96b8d3e1d5f741a09dc3700db732a7ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e] to archive 2024-11-21T00:28:21,608 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:28:21,610 WARN [regionserver/5ed4808ef0e6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-21T00:28:21,616 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3d973ccba3df4636abe9df9ae7f40fa8 2024-11-21T00:28:21,617 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/190767549a164c7582f3103d05cc87bd 2024-11-21T00:28:21,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/11364b0efa47416cb2a15a36ceb1186f 2024-11-21T00:28:21,619 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/36d88c536518438ea9c497b16d5ae5b2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/36d88c536518438ea9c497b16d5ae5b2 2024-11-21T00:28:21,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/30b25fdf27fd4bfc813700e7dd4951d0 2024-11-21T00:28:21,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d1ce90361ef54dfa8eed78f089ec91af 2024-11-21T00:28:21,628 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/97aa0d5e14cb44bab4ebaca45a189499 2024-11-21T00:28:21,629 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/05e73b0a1e1148a5983936e4f27fffb0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/05e73b0a1e1148a5983936e4f27fffb0 2024-11-21T00:28:21,630 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/bd33c08429564818b87f5f912ac9ae0c 2024-11-21T00:28:21,634 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96fb0b3f0e90451ab901633214e25c39 2024-11-21T00:28:21,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/33aa7911c23c405f8396fa0287776433 2024-11-21T00:28:21,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/15a9919f05f846a3b15886ceaa791e46 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/15a9919f05f846a3b15886ceaa791e46 2024-11-21T00:28:21,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/f3687d763348461fbf6f63a605b539a7 2024-11-21T00:28:21,637 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d7688dd7394a419c819bba8e5764522e 2024-11-21T00:28:21,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/90d93621476648ce82a54e400600b092 2024-11-21T00:28:21,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/a9c7dc17732643bda2d4aa01351a50a7 2024-11-21T00:28:21,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/8bd330880a7046deab132fd85213c20b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/8bd330880a7046deab132fd85213c20b 2024-11-21T00:28:21,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ed52eb9432d945a88c78550484d5e50a 2024-11-21T00:28:21,641 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/db19eb80a5d54677aa798c29e581c369 2024-11-21T00:28:21,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/0913e150365b48548c5ac302f19eeb62 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/0913e150365b48548c5ac302f19eeb62 2024-11-21T00:28:21,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/c05851418ce345acbec6c1921288a81a 2024-11-21T00:28:21,657 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d20d982765aa49a5a2acabc099947a60 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/d20d982765aa49a5a2acabc099947a60 2024-11-21T00:28:21,658 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/6ce8d4a4e6e64d47a2ff50bf390f35d3 2024-11-21T00:28:21,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/98fc6cc9a33b48bf99a6f0c778a515c0 2024-11-21T00:28:21,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/ecf5f22b6d094284b5f8b36509cfbb0f 2024-11-21T00:28:21,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/3b23a755a03a49fd98d425bb043c81fd 2024-11-21T00:28:21,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96b8d3e1d5f741a09dc3700db732a7ee to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/96b8d3e1d5f741a09dc3700db732a7ee 2024-11-21T00:28:21,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/1ae8f16adc5149c68b3997748b17d6e9 2024-11-21T00:28:21,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/702ea2d63f4348bc95dc18b9aafb7d2e 2024-11-21T00:28:21,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f973e4cdf4d244c3acecd845ccd2b976, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8c1db950410c448596eebbf2bbb6876c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/965417276d0e4b41ae8c061d4e27765f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1295cccdab8e4291914a62820b883115, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/126a652f6f834fc3a5ada30211649e42, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5e92584dae824cbd86e8cab328e102df, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d24ab0aa764c45aeaf14ee5b770d536f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/6d9d11be3b454b5697fd9c0492693166, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/a99f962f0043451993ccfb092908e72a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/844ce6ade75a4b9fb4a17b18e6117439, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/931258e261cc42c1b42867b0c3560d9c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/82a50ab151c14cf2a79286a3cd42c8d8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/962fab00ba9349e3ad66d7bbf67f0223, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/c3a185cb69d14b698a3fcd4e53db4ee2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f91dd6bce8ac4623aa0794f50c41bfdc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5614d4d828ef4bf7a2b1d4742e9340ac, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/e40e944025b54cc78393235de7506078, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8584621b45c1464eb28c10fd8aab79ee, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8168cbae684e4d0cbbf14f92001804ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/bb9c007cd9bb4acca37916c22865af81, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/455836cd58084e57ac0663f32431b7d7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5a8cec96fa154295a0fc3de7cdeb1309, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5646a28171564e488822226b79e14e29, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/36bb5876121e4c56a9ad39bc80ef23a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1ab2b8932b514f0c9a683a0b236b6e57, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/2ca7b68b50244e389dfe2753c9c44384, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/4cc959bebaa24abbb56fe730489faa38, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f1f378114dc644f0ae96df881cf2c2c1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/29c3fab8ccb3462abc0eaee88cb78280] to archive 2024-11-21T00:28:21,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:28:21,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f973e4cdf4d244c3acecd845ccd2b976 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f973e4cdf4d244c3acecd845ccd2b976 2024-11-21T00:28:21,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8c1db950410c448596eebbf2bbb6876c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8c1db950410c448596eebbf2bbb6876c 2024-11-21T00:28:21,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/965417276d0e4b41ae8c061d4e27765f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/965417276d0e4b41ae8c061d4e27765f 2024-11-21T00:28:21,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1295cccdab8e4291914a62820b883115 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1295cccdab8e4291914a62820b883115 2024-11-21T00:28:21,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/126a652f6f834fc3a5ada30211649e42 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/126a652f6f834fc3a5ada30211649e42 2024-11-21T00:28:21,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5e92584dae824cbd86e8cab328e102df to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5e92584dae824cbd86e8cab328e102df 2024-11-21T00:28:21,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d24ab0aa764c45aeaf14ee5b770d536f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d24ab0aa764c45aeaf14ee5b770d536f 2024-11-21T00:28:21,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/6d9d11be3b454b5697fd9c0492693166 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/6d9d11be3b454b5697fd9c0492693166 2024-11-21T00:28:21,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/a99f962f0043451993ccfb092908e72a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/a99f962f0043451993ccfb092908e72a 2024-11-21T00:28:21,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/844ce6ade75a4b9fb4a17b18e6117439 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/844ce6ade75a4b9fb4a17b18e6117439 2024-11-21T00:28:21,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/931258e261cc42c1b42867b0c3560d9c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/931258e261cc42c1b42867b0c3560d9c 2024-11-21T00:28:21,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/82a50ab151c14cf2a79286a3cd42c8d8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/82a50ab151c14cf2a79286a3cd42c8d8 2024-11-21T00:28:21,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/962fab00ba9349e3ad66d7bbf67f0223 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/962fab00ba9349e3ad66d7bbf67f0223 2024-11-21T00:28:21,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/c3a185cb69d14b698a3fcd4e53db4ee2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/c3a185cb69d14b698a3fcd4e53db4ee2 2024-11-21T00:28:21,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f91dd6bce8ac4623aa0794f50c41bfdc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f91dd6bce8ac4623aa0794f50c41bfdc 2024-11-21T00:28:21,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5614d4d828ef4bf7a2b1d4742e9340ac to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5614d4d828ef4bf7a2b1d4742e9340ac 2024-11-21T00:28:21,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/e40e944025b54cc78393235de7506078 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/e40e944025b54cc78393235de7506078 2024-11-21T00:28:21,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8584621b45c1464eb28c10fd8aab79ee to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8584621b45c1464eb28c10fd8aab79ee 2024-11-21T00:28:21,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8168cbae684e4d0cbbf14f92001804ed to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/8168cbae684e4d0cbbf14f92001804ed 2024-11-21T00:28:21,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/bb9c007cd9bb4acca37916c22865af81 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/bb9c007cd9bb4acca37916c22865af81 2024-11-21T00:28:21,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/455836cd58084e57ac0663f32431b7d7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/455836cd58084e57ac0663f32431b7d7 2024-11-21T00:28:21,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5a8cec96fa154295a0fc3de7cdeb1309 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5a8cec96fa154295a0fc3de7cdeb1309 2024-11-21T00:28:21,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5646a28171564e488822226b79e14e29 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/5646a28171564e488822226b79e14e29 2024-11-21T00:28:21,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/36bb5876121e4c56a9ad39bc80ef23a6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/36bb5876121e4c56a9ad39bc80ef23a6 2024-11-21T00:28:21,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1ab2b8932b514f0c9a683a0b236b6e57 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/1ab2b8932b514f0c9a683a0b236b6e57 2024-11-21T00:28:21,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/2ca7b68b50244e389dfe2753c9c44384 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/2ca7b68b50244e389dfe2753c9c44384 2024-11-21T00:28:21,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/4cc959bebaa24abbb56fe730489faa38 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/4cc959bebaa24abbb56fe730489faa38 2024-11-21T00:28:21,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f1f378114dc644f0ae96df881cf2c2c1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f1f378114dc644f0ae96df881cf2c2c1 2024-11-21T00:28:21,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/29c3fab8ccb3462abc0eaee88cb78280 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/29c3fab8ccb3462abc0eaee88cb78280 2024-11-21T00:28:21,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/06c73f682cb74fc6b5e91a4619d0ef43, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e226072c28eb40dfbb926f90bfe8a666, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/9d136a9af266416fa7d886319d60e56e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5b88c35d739f43c1a2e19032a69b51cd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/641cf3988b914dc3b09dba40e7a08296, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a00b0e109f824b85b2b7f79c3ee799a4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/edafce3994c2432eaefff174cdfffed7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b8b3b063aefa40e7b6ba98acf8ad084b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/537be39614cc4fa1972c4ff18d7e3cb8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/af44fa521a5d4af99d94280cc3a08d1d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/2f142db6b0464af9a937e5eee3f389ae, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e689490997684461a54de56aca24371a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/74ded48898594a94b54de3da4ea615a4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/46965fb3fc1d4c9c8b17c2ff568c4823, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d534fd29245c4e37b349f5e41b8b3465, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/681991cbe39d4fa8b7881a49f4a5f7d9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a42a75066964478dbae35812e94692ba, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/0d8a2215ad464ee7b8663a3bbc099cd3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e66a99cdc54a4c6ca49d0e1b0be95591, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b9ade6934a894d01b767d991f1d049c4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/32b5a0fe2fd74a0c871e19f91be0d175, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d949330cd18f49b095f4e1a8c029de4e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/59d25f4c40414204aff28dfdbf03bde9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5339bf5692254dba8e2caba233f39208, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5e38bfb5495b4ad4801c51f2c84e5096, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/c097bef542f441cd9678173e06fdddcc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e4963007fcee4d15a28a3f547c344158, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/69933788998f47798b31f2e47a74f0fa] to archive 2024-11-21T00:28:21,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:28:21,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/06c73f682cb74fc6b5e91a4619d0ef43 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/06c73f682cb74fc6b5e91a4619d0ef43 2024-11-21T00:28:21,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e226072c28eb40dfbb926f90bfe8a666 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e226072c28eb40dfbb926f90bfe8a666 2024-11-21T00:28:21,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/9d136a9af266416fa7d886319d60e56e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/9d136a9af266416fa7d886319d60e56e 2024-11-21T00:28:21,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5b88c35d739f43c1a2e19032a69b51cd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5b88c35d739f43c1a2e19032a69b51cd 2024-11-21T00:28:21,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/641cf3988b914dc3b09dba40e7a08296 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/641cf3988b914dc3b09dba40e7a08296 2024-11-21T00:28:21,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a00b0e109f824b85b2b7f79c3ee799a4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a00b0e109f824b85b2b7f79c3ee799a4 2024-11-21T00:28:21,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/edafce3994c2432eaefff174cdfffed7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/edafce3994c2432eaefff174cdfffed7 2024-11-21T00:28:21,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b8b3b063aefa40e7b6ba98acf8ad084b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b8b3b063aefa40e7b6ba98acf8ad084b 2024-11-21T00:28:21,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/6e75e3e7b70e4dfaaaa2dc4ef8976f47 2024-11-21T00:28:21,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/537be39614cc4fa1972c4ff18d7e3cb8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/537be39614cc4fa1972c4ff18d7e3cb8 2024-11-21T00:28:21,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/af44fa521a5d4af99d94280cc3a08d1d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/af44fa521a5d4af99d94280cc3a08d1d 2024-11-21T00:28:21,804 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/2f142db6b0464af9a937e5eee3f389ae to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/2f142db6b0464af9a937e5eee3f389ae 2024-11-21T00:28:21,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e689490997684461a54de56aca24371a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e689490997684461a54de56aca24371a 2024-11-21T00:28:21,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/74ded48898594a94b54de3da4ea615a4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/74ded48898594a94b54de3da4ea615a4 2024-11-21T00:28:21,808 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/46965fb3fc1d4c9c8b17c2ff568c4823 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/46965fb3fc1d4c9c8b17c2ff568c4823 2024-11-21T00:28:21,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d534fd29245c4e37b349f5e41b8b3465 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d534fd29245c4e37b349f5e41b8b3465 2024-11-21T00:28:21,810 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/681991cbe39d4fa8b7881a49f4a5f7d9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/681991cbe39d4fa8b7881a49f4a5f7d9 2024-11-21T00:28:21,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a42a75066964478dbae35812e94692ba to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/a42a75066964478dbae35812e94692ba 2024-11-21T00:28:21,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/0d8a2215ad464ee7b8663a3bbc099cd3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/0d8a2215ad464ee7b8663a3bbc099cd3 2024-11-21T00:28:21,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e66a99cdc54a4c6ca49d0e1b0be95591 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e66a99cdc54a4c6ca49d0e1b0be95591 2024-11-21T00:28:21,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b9ade6934a894d01b767d991f1d049c4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b9ade6934a894d01b767d991f1d049c4 2024-11-21T00:28:21,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/32b5a0fe2fd74a0c871e19f91be0d175 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/32b5a0fe2fd74a0c871e19f91be0d175 2024-11-21T00:28:21,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d949330cd18f49b095f4e1a8c029de4e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/d949330cd18f49b095f4e1a8c029de4e 2024-11-21T00:28:21,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/59d25f4c40414204aff28dfdbf03bde9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/59d25f4c40414204aff28dfdbf03bde9 2024-11-21T00:28:21,827 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5339bf5692254dba8e2caba233f39208 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5339bf5692254dba8e2caba233f39208 2024-11-21T00:28:21,828 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5e38bfb5495b4ad4801c51f2c84e5096 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/5e38bfb5495b4ad4801c51f2c84e5096 2024-11-21T00:28:21,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/c097bef542f441cd9678173e06fdddcc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/c097bef542f441cd9678173e06fdddcc 2024-11-21T00:28:21,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e4963007fcee4d15a28a3f547c344158 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/e4963007fcee4d15a28a3f547c344158 2024-11-21T00:28:21,831 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5ed4808ef0e6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/69933788998f47798b31f2e47a74f0fa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/69933788998f47798b31f2e47a74f0fa 2024-11-21T00:28:21,993 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/9536ea0b13a0427e8da04d83644b54bd 2024-11-21T00:28:22,001 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/543f4a470575422eb05bc55a05a55ac9 is 50, key is test_row_2/B:col10/1732148899002/Put/seqid=0 2024-11-21T00:28:22,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742298_1474 (size=7415) 2024-11-21T00:28:22,413 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/543f4a470575422eb05bc55a05a55ac9 2024-11-21T00:28:22,419 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/4372c2010f6d4c40ad47976657081d0b is 50, key is test_row_2/C:col10/1732148899002/Put/seqid=0 2024-11-21T00:28:22,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742299_1475 (size=7415) 2024-11-21T00:28:22,824 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/4372c2010f6d4c40ad47976657081d0b 2024-11-21T00:28:22,832 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/A/9536ea0b13a0427e8da04d83644b54bd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/9536ea0b13a0427e8da04d83644b54bd 2024-11-21T00:28:22,838 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/9536ea0b13a0427e8da04d83644b54bd, entries=50, sequenceid=459, filesize=13.5 K 2024-11-21T00:28:22,839 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/B/543f4a470575422eb05bc55a05a55ac9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/543f4a470575422eb05bc55a05a55ac9 2024-11-21T00:28:22,843 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/543f4a470575422eb05bc55a05a55ac9, entries=50, sequenceid=459, filesize=7.2 K 2024-11-21T00:28:22,844 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/.tmp/C/4372c2010f6d4c40ad47976657081d0b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/4372c2010f6d4c40ad47976657081d0b 2024-11-21T00:28:22,849 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/4372c2010f6d4c40ad47976657081d0b, entries=50, sequenceid=459, filesize=7.2 K 2024-11-21T00:28:22,857 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5776037b580ebf80c0ab7fa6d7225dd4 in 1827ms, sequenceid=459, compaction requested=true 2024-11-21T00:28:22,868 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=4 2024-11-21T00:28:22,868 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. 2024-11-21T00:28:22,868 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 5776037b580ebf80c0ab7fa6d7225dd4: Waiting for close lock at 1732148899365Running coprocessor pre-close hooks at 1732148899365Disabling compacts and flushes for region at 1732148899365Disabling writes for close at 1732148901030 (+1665 ms)Obtaining lock to block concurrent updates at 1732148901030Preparing flush snapshotting stores in 5776037b580ebf80c0ab7fa6d7225dd4 at 1732148901030Finished memstore snapshotting TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732148901030Flushing stores of TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4. at 1732148901031 (+1 ms)Flushing 5776037b580ebf80c0ab7fa6d7225dd4/A: creating writer at 1732148901031Flushing mob file 5776037b580ebf80c0ab7fa6d7225dd4/A: appending metadata at 1732148901069 (+38 ms)Flushing mob file 5776037b580ebf80c0ab7fa6d7225dd4/A: closing flushed file at 1732148901069Flushing 5776037b580ebf80c0ab7fa6d7225dd4/A: appending metadata at 1732148901580 (+511 ms)Flushing 5776037b580ebf80c0ab7fa6d7225dd4/A: closing flushed file at 1732148901580Flushing 5776037b580ebf80c0ab7fa6d7225dd4/B: creating writer at 1732148901998 (+418 ms)Flushing 5776037b580ebf80c0ab7fa6d7225dd4/B: appending metadata at 1732148902000 (+2 ms)Flushing 5776037b580ebf80c0ab7fa6d7225dd4/B: closing flushed file at 1732148902000Flushing 5776037b580ebf80c0ab7fa6d7225dd4/C: creating writer at 1732148902416 (+416 ms)Flushing 5776037b580ebf80c0ab7fa6d7225dd4/C: appending metadata at 1732148902418 (+2 ms)Flushing 5776037b580ebf80c0ab7fa6d7225dd4/C: closing flushed file at 1732148902418Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e950065: reopening flushed file at 1732148902829 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2716266c: reopening flushed file at 1732148902838 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5033fe8c: reopening flushed file at 1732148902843 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5776037b580ebf80c0ab7fa6d7225dd4 in 1827ms, sequenceid=459, compaction requested=true at 1732148902857 (+14 ms)Writing region close event to WAL at 1732148902863 (+6 ms)Running coprocessor post-close hooks at 1732148902868 (+5 ms)Closed at 1732148902868 2024-11-21T00:28:22,871 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=5776037b580ebf80c0ab7fa6d7225dd4, regionState=CLOSED 2024-11-21T00:28:22,876 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:22,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:28:22,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T00:28:22,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 5776037b580ebf80c0ab7fa6d7225dd4, server=5ed4808ef0e6,38589,1732148778087 in 3.6700 sec 2024-11-21T00:28:22,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-21T00:28:22,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5776037b580ebf80c0ab7fa6d7225dd4, UNASSIGN in 3.6780 sec 2024-11-21T00:28:22,887 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148902887"}]},"ts":"1732148902887"} 2024-11-21T00:28:22,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T00:28:22,889 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:28:22,889 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:28:22,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.6860 sec 2024-11-21T00:28:22,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.7340 sec 2024-11-21T00:28:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T00:28:23,342 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:23,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:28:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,350 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,352 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T00:28:23,357 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/recovered.edits] 2024-11-21T00:28:23,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/4a84f4f89b684ac8b9f5607f37b62a82 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/4a84f4f89b684ac8b9f5607f37b62a82 2024-11-21T00:28:23,378 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/9536ea0b13a0427e8da04d83644b54bd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/9536ea0b13a0427e8da04d83644b54bd 2024-11-21T00:28:23,383 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/dd0c158f975e48d7b6191ea813bbccdf to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/A/dd0c158f975e48d7b6191ea813bbccdf 2024-11-21T00:28:23,387 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/543f4a470575422eb05bc55a05a55ac9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/543f4a470575422eb05bc55a05a55ac9 2024-11-21T00:28:23,391 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d701681e1f1842eaabc98be1b5255d24 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/d701681e1f1842eaabc98be1b5255d24 2024-11-21T00:28:23,397 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f540d3dbeea043d0bc04648625186da4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/B/f540d3dbeea043d0bc04648625186da4 2024-11-21T00:28:23,404 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/4372c2010f6d4c40ad47976657081d0b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/4372c2010f6d4c40ad47976657081d0b 2024-11-21T00:28:23,405 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/93ed393bbc784c8a8cca72d20f1982d7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/93ed393bbc784c8a8cca72d20f1982d7 2024-11-21T00:28:23,407 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b56fe9ea4123492583a4876815330a0e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/C/b56fe9ea4123492583a4876815330a0e 2024-11-21T00:28:23,436 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/recovered.edits/462.seqid to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4/recovered.edits/462.seqid 2024-11-21T00:28:23,437 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,439 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:28:23,439 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:28:23,441 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T00:28:23,446 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211438a24e6107464d976babcc7d076e3e_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211438a24e6107464d976babcc7d076e3e_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,451 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112115dd2b90d91647e2a19dee2812fc99e2_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112115dd2b90d91647e2a19dee2812fc99e2_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,452 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121233b30e578734ee7a7a3479d67831ce7_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121233b30e578734ee7a7a3479d67831ce7_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T00:28:23,474 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121260e8f7c889c4d4e99065482ede03de6_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121260e8f7c889c4d4e99065482ede03de6_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,475 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112132bb6c446b924ef1a1225bbef3596306_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112132bb6c446b924ef1a1225bbef3596306_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,476 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213d8b48cb9fe34cfca51b283fb6247a8d_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213d8b48cb9fe34cfca51b283fb6247a8d_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,477 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112140688e09af9c4cfca04a8a84d888f134_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112140688e09af9c4cfca04a8a84d888f134_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,480 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112151028109d32242129b79b55de9e53bd8_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112151028109d32242129b79b55de9e53bd8_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,490 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215dac43f559734e809b1cac56d94d80b0_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215dac43f559734e809b1cac56d94d80b0_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,491 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112162a3dfca8d5243cd8338758ba3dcb5ef_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112162a3dfca8d5243cd8338758ba3dcb5ef_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,492 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216f55643bb931496e93b516eebeb2e35a_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216f55643bb931496e93b516eebeb2e35a_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,494 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218eb6ff3f951240099ed39388e2f0c215_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218eb6ff3f951240099ed39388e2f0c215_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,495 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112194b0ed9e096c4910b1af781958003410_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112194b0ed9e096c4910b1af781958003410_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,496 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a5538070be71467389e35957f74d2995_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a5538070be71467389e35957f74d2995_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,497 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6124f8d136d48448cc927866894afd8_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6124f8d136d48448cc927866894afd8_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,498 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a72b2b4f210a4c4cb679350df0fb6dd8_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a72b2b4f210a4c4cb679350df0fb6dd8_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,500 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a893dbda0a20408e8544194c160379c6_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a893dbda0a20408e8544194c160379c6_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,501 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b387f28417f04496ab33df93da85eccc_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b387f28417f04496ab33df93da85eccc_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,502 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bb4d05f38017496d81e6119ba8c2fd36_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bb4d05f38017496d81e6119ba8c2fd36_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,503 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d2f43f50a05a42549961778497205c86_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d2f43f50a05a42549961778497205c86_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,504 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6a5a312f37b4e2ba28dcd0cabc590b6_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d6a5a312f37b4e2ba28dcd0cabc590b6_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121daf58a4f13614b348ac5865e86ce54cc_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121daf58a4f13614b348ac5865e86ce54cc_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e175884c9b94475eba8bef673feb67e0_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e175884c9b94475eba8bef673feb67e0_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eb8bd63a4a5244e790975a61cd10a818_5776037b580ebf80c0ab7fa6d7225dd4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121eb8bd63a4a5244e790975a61cd10a818_5776037b580ebf80c0ab7fa6d7225dd4 2024-11-21T00:28:23,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:28:23,536 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,539 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:28:23,541 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:28:23,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,544 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:28:23,544 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148903544"}]},"ts":"9223372036854775807"} 2024-11-21T00:28:23,546 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:28:23,546 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5776037b580ebf80c0ab7fa6d7225dd4, NAME => 'TestAcidGuarantees,,1732148876616.5776037b580ebf80c0ab7fa6d7225dd4.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:28:23,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:28:23,546 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148903546"}]},"ts":"9223372036854775807"} 2024-11-21T00:28:23,548 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:28:23,549 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 206 msec 2024-11-21T00:28:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T00:28:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:28:23,675 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:23,690 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=940 (was 991), ProcessCount=11 (was 11), AvailableMemoryMB=3036 (was 265) - AvailableMemoryMB LEAK? - 2024-11-21T00:28:23,721 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=940, ProcessCount=11, AvailableMemoryMB=3032 2024-11-21T00:28:23,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:28:23,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:28:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:23,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:28:23,736 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:23,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:28:23,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-21T00:28:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-21T00:28:23,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742300_1476 (size=959) 2024-11-21T00:28:23,752 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:28:23,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742301_1477 (size=53) 2024-11-21T00:28:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-21T00:28:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-21T00:28:24,168 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:28:24,169 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5151f743f103348045956281ec3b4bb9, disabling compactions & flushes 2024-11-21T00:28:24,169 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,169 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,169 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. after waiting 0 ms 2024-11-21T00:28:24,169 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,169 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,169 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5151f743f103348045956281ec3b4bb9: Waiting for close lock at 1732148904169Disabling compacts and flushes for region at 1732148904169Disabling writes for close at 1732148904169Writing region close event to WAL at 1732148904169Closed at 1732148904169 2024-11-21T00:28:24,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:28:24,171 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148904170"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148904170"}]},"ts":"1732148904170"} 2024-11-21T00:28:24,173 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:28:24,184 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:28:24,185 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148904184"}]},"ts":"1732148904184"} 2024-11-21T00:28:24,190 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:28:24,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, ASSIGN}] 2024-11-21T00:28:24,192 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, ASSIGN 2024-11-21T00:28:24,193 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=false 2024-11-21T00:28:24,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=5151f743f103348045956281ec3b4bb9, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:24,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, ASSIGN because future has completed 2024-11-21T00:28:24,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:28:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-21T00:28:24,499 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,499 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:28:24,500 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,500 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:28:24,500 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,500 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,501 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,502 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5151f743f103348045956281ec3b4bb9 columnFamilyName A 2024-11-21T00:28:24,502 DEBUG [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:24,502 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] regionserver.HStore(327): Store=5151f743f103348045956281ec3b4bb9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:24,503 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,503 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5151f743f103348045956281ec3b4bb9 columnFamilyName B 2024-11-21T00:28:24,503 DEBUG [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:24,504 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] regionserver.HStore(327): Store=5151f743f103348045956281ec3b4bb9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:24,504 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,505 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5151f743f103348045956281ec3b4bb9 columnFamilyName C 2024-11-21T00:28:24,505 DEBUG [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:24,505 INFO [StoreOpener-5151f743f103348045956281ec3b4bb9-1 {}] regionserver.HStore(327): Store=5151f743f103348045956281ec3b4bb9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:24,506 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,506 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,508 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,509 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,509 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,509 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:28:24,510 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,512 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:28:24,512 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 5151f743f103348045956281ec3b4bb9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74007880, jitterRate=0.10280334949493408}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:28:24,512 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:24,513 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 5151f743f103348045956281ec3b4bb9: Running coprocessor pre-open hook at 1732148904500Writing region info on filesystem at 1732148904500Initializing all the Stores at 1732148904501 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148904501Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148904501Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148904501Cleaning up temporary data from old regions at 1732148904509 (+8 ms)Running coprocessor post-open hooks at 1732148904512 (+3 ms)Region opened successfully at 1732148904512 2024-11-21T00:28:24,513 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., pid=114, masterSystemTime=1732148904497 2024-11-21T00:28:24,515 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,515 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:24,515 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=5151f743f103348045956281ec3b4bb9, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:24,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:28:24,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T00:28:24,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 in 172 msec 2024-11-21T00:28:24,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-21T00:28:24,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, ASSIGN in 328 msec 2024-11-21T00:28:24,520 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:28:24,520 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148904520"}]},"ts":"1732148904520"} 2024-11-21T00:28:24,521 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:28:24,521 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:28:24,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 796 msec 2024-11-21T00:28:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-21T00:28:24,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:24,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@389d3e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,886 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ca117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387cba0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:24,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:24,895 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:28:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:28:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8b0a928 2024-11-21T00:28:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:28:24,898 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:28:24,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8dcada, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6712880d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,915 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fed3cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e17b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,920 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a65c645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,922 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cffa173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,927 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d6c109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e873cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,942 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a385c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,944 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e650cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,950 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5638b05c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,954 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1968b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,965 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be19faa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e677b7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,982 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f002ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fafbf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:24,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62424f2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:24,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471a03b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:24,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:24,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:25,001 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e12f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:25,005 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:25,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c90d17e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:25,010 DEBUG [Thread-2134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,016 DEBUG [Thread-2145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-21T00:28:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:25,028 DEBUG [Thread-2147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,029 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:25,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:25,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:25,036 DEBUG [Thread-2132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,036 DEBUG [Thread-2130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,051 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,056 DEBUG [Thread-2139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,059 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:25,059 DEBUG [Thread-2139 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:25,059 DEBUG [Thread-2139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:25,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,064 DEBUG [Thread-2128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,079 DEBUG [Thread-2143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,089 DEBUG [Thread-2141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:25,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2] 2024-11-21T00:28:25,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/7efe8a6374124cdd9d7e1c384935dedc is 50, key is test_row_0/A:col10/1732148905058/Put/seqid=0 2024-11-21T00:28:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:25,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148965124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742302_1478 (size=14341) 2024-11-21T00:28:25,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/7efe8a6374124cdd9d7e1c384935dedc 2024-11-21T00:28:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148965142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148965142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148965143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148965153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/4d1720e1e61c4a16b0fcbaa701cfbe5a is 50, key is test_row_0/B:col10/1732148905058/Put/seqid=0 2024-11-21T00:28:25,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742303_1479 (size=12001) 2024-11-21T00:28:25,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/4d1720e1e61c4a16b0fcbaa701cfbe5a 2024-11-21T00:28:25,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148965252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148965273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148965273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148965276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148965276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/87add02bf35f4d6888e64fdb4142172c is 50, key is test_row_0/C:col10/1732148905058/Put/seqid=0 2024-11-21T00:28:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:25,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742304_1480 (size=12001) 2024-11-21T00:28:25,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/87add02bf35f4d6888e64fdb4142172c 2024-11-21T00:28:25,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/7efe8a6374124cdd9d7e1c384935dedc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7efe8a6374124cdd9d7e1c384935dedc 2024-11-21T00:28:25,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7efe8a6374124cdd9d7e1c384935dedc, entries=200, sequenceid=13, filesize=14.0 K 2024-11-21T00:28:25,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/4d1720e1e61c4a16b0fcbaa701cfbe5a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4d1720e1e61c4a16b0fcbaa701cfbe5a 2024-11-21T00:28:25,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4d1720e1e61c4a16b0fcbaa701cfbe5a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:28:25,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/87add02bf35f4d6888e64fdb4142172c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/87add02bf35f4d6888e64fdb4142172c 2024-11-21T00:28:25,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/87add02bf35f4d6888e64fdb4142172c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T00:28:25,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5151f743f103348045956281ec3b4bb9 in 385ms, sequenceid=13, compaction requested=false 2024-11-21T00:28:25,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:25,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:28:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:25,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/8c83cfc17ed541b1bc220ad00412b8a2 is 50, key is test_row_0/A:col10/1732148905106/Put/seqid=0 2024-11-21T00:28:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148965522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148965524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148965526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148965528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742305_1481 (size=14341) 2024-11-21T00:28:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148965542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/8c83cfc17ed541b1bc220ad00412b8a2 2024-11-21T00:28:25,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/eaaca4ff49674e53b77c4be5007f7610 is 50, key is test_row_0/B:col10/1732148905106/Put/seqid=0 2024-11-21T00:28:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:25,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148965653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148965655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148965655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742306_1482 (size=12001) 2024-11-21T00:28:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148965663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148965663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148965873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148965874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148965874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148965882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148965882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:25,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:25,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:26,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/eaaca4ff49674e53b77c4be5007f7610 2024-11-21T00:28:26,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/8e7762731f85471da1d51c98eab2d142 is 50, key is test_row_0/C:col10/1732148905106/Put/seqid=0 2024-11-21T00:28:26,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:26,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742307_1483 (size=12001) 2024-11-21T00:28:26,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:26,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:26,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/8e7762731f85471da1d51c98eab2d142 2024-11-21T00:28:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:26,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/8c83cfc17ed541b1bc220ad00412b8a2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/8c83cfc17ed541b1bc220ad00412b8a2 2024-11-21T00:28:26,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148966182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148966197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148966199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148966197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148966202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/8c83cfc17ed541b1bc220ad00412b8a2, entries=200, sequenceid=38, filesize=14.0 K 2024-11-21T00:28:26,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/eaaca4ff49674e53b77c4be5007f7610 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/eaaca4ff49674e53b77c4be5007f7610 2024-11-21T00:28:26,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/eaaca4ff49674e53b77c4be5007f7610, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T00:28:26,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/8e7762731f85471da1d51c98eab2d142 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/8e7762731f85471da1d51c98eab2d142 2024-11-21T00:28:26,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/8e7762731f85471da1d51c98eab2d142, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T00:28:26,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5151f743f103348045956281ec3b4bb9 in 761ms, sequenceid=38, compaction requested=false 2024-11-21T00:28:26,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:26,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T00:28:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:26,310 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/be1f0bd3246e44eba9db2b82d096f1d1 is 50, key is test_row_0/A:col10/1732148905521/Put/seqid=0 2024-11-21T00:28:26,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742308_1484 (size=12001) 2024-11-21T00:28:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:26,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:26,785 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/be1f0bd3246e44eba9db2b82d096f1d1 2024-11-21T00:28:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148966790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148966795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/29039000a9254a74bc7ce2d3b6408867 is 50, key is test_row_0/B:col10/1732148905521/Put/seqid=0 2024-11-21T00:28:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148966799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148966801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148966808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742309_1485 (size=12001) 2024-11-21T00:28:26,866 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/29039000a9254a74bc7ce2d3b6408867 2024-11-21T00:28:26,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/b36e9b3edc9c4ea29463db1f7ce8ba66 is 50, key is test_row_0/C:col10/1732148905521/Put/seqid=0 2024-11-21T00:28:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148966912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148966928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148966933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148966933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148966940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742310_1486 (size=12001) 2024-11-21T00:28:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148967136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148967153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148967153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148967155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:27,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148967160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,358 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/b36e9b3edc9c4ea29463db1f7ce8ba66 2024-11-21T00:28:27,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/be1f0bd3246e44eba9db2b82d096f1d1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/be1f0bd3246e44eba9db2b82d096f1d1 2024-11-21T00:28:27,373 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/be1f0bd3246e44eba9db2b82d096f1d1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T00:28:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/29039000a9254a74bc7ce2d3b6408867 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/29039000a9254a74bc7ce2d3b6408867 2024-11-21T00:28:27,382 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/29039000a9254a74bc7ce2d3b6408867, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T00:28:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/b36e9b3edc9c4ea29463db1f7ce8ba66 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b36e9b3edc9c4ea29463db1f7ce8ba66 2024-11-21T00:28:27,388 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b36e9b3edc9c4ea29463db1f7ce8ba66, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T00:28:27,390 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5151f743f103348045956281ec3b4bb9 in 1080ms, sequenceid=49, compaction requested=true 2024-11-21T00:28:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-21T00:28:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-21T00:28:27,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T00:28:27,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3620 sec 2024-11-21T00:28:27,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.3700 sec 2024-11-21T00:28:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:27,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:27,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:28:27,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/bfa39c3ff42e458b86d1551b03b76313 is 50, key is test_row_0/A:col10/1732148906794/Put/seqid=0 2024-11-21T00:28:27,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742311_1487 (size=14341) 2024-11-21T00:28:27,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/bfa39c3ff42e458b86d1551b03b76313 2024-11-21T00:28:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148967516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148967521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/7a7f430638b84f409f13d48f1b384d04 is 50, key is test_row_0/B:col10/1732148906794/Put/seqid=0 2024-11-21T00:28:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148967548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148967549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148967551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742312_1488 (size=12001) 2024-11-21T00:28:27,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/7a7f430638b84f409f13d48f1b384d04 2024-11-21T00:28:27,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/749af46beb734dfaa757838fb8d08611 is 50, key is test_row_0/C:col10/1732148906794/Put/seqid=0 2024-11-21T00:28:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742313_1489 (size=12001) 2024-11-21T00:28:27,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/749af46beb734dfaa757838fb8d08611 2024-11-21T00:28:27,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/bfa39c3ff42e458b86d1551b03b76313 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/bfa39c3ff42e458b86d1551b03b76313 2024-11-21T00:28:27,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/bfa39c3ff42e458b86d1551b03b76313, entries=200, sequenceid=75, filesize=14.0 K 2024-11-21T00:28:27,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/7a7f430638b84f409f13d48f1b384d04 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/7a7f430638b84f409f13d48f1b384d04 2024-11-21T00:28:27,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/7a7f430638b84f409f13d48f1b384d04, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T00:28:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148967664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/749af46beb734dfaa757838fb8d08611 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/749af46beb734dfaa757838fb8d08611 2024-11-21T00:28:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/749af46beb734dfaa757838fb8d08611, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T00:28:27,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5151f743f103348045956281ec3b4bb9 in 205ms, sequenceid=75, compaction requested=true 2024-11-21T00:28:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:27,685 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:27,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:27,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:27,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:27,686 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:27,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:27,687 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:27,687 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:27,687 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:27,687 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:27,687 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:27,687 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:27,687 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4d1720e1e61c4a16b0fcbaa701cfbe5a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/eaaca4ff49674e53b77c4be5007f7610, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/29039000a9254a74bc7ce2d3b6408867, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/7a7f430638b84f409f13d48f1b384d04] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=46.9 K 2024-11-21T00:28:27,688 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1720e1e61c4a16b0fcbaa701cfbe5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148905056 2024-11-21T00:28:27,688 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7efe8a6374124cdd9d7e1c384935dedc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/8c83cfc17ed541b1bc220ad00412b8a2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/be1f0bd3246e44eba9db2b82d096f1d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/bfa39c3ff42e458b86d1551b03b76313] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=53.7 K 2024-11-21T00:28:27,688 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting eaaca4ff49674e53b77c4be5007f7610, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732148905106 2024-11-21T00:28:27,688 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7efe8a6374124cdd9d7e1c384935dedc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148905049 2024-11-21T00:28:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:27,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 29039000a9254a74bc7ce2d3b6408867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732148905521 2024-11-21T00:28:27,689 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c83cfc17ed541b1bc220ad00412b8a2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732148905106 2024-11-21T00:28:27,689 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting be1f0bd3246e44eba9db2b82d096f1d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732148905521 2024-11-21T00:28:27,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a7f430638b84f409f13d48f1b384d04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732148906794 2024-11-21T00:28:27,690 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa39c3ff42e458b86d1551b03b76313, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732148906794 2024-11-21T00:28:27,699 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:27,699 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a58983ae938144e1810007e1cbdb7578 is 50, key is test_row_0/A:col10/1732148906794/Put/seqid=0 2024-11-21T00:28:27,708 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:27,709 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c97b2e23055b43c6b6e749281a4094f0 is 50, key is test_row_0/B:col10/1732148906794/Put/seqid=0 2024-11-21T00:28:27,709 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:27,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:27,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/68cc4db6c1434d6caafcbe675e1cab14 is 50, key is test_row_0/A:col10/1732148907526/Put/seqid=0 2024-11-21T00:28:27,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742315_1491 (size=12139) 2024-11-21T00:28:27,770 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c97b2e23055b43c6b6e749281a4094f0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c97b2e23055b43c6b6e749281a4094f0 2024-11-21T00:28:27,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742314_1490 (size=12139) 2024-11-21T00:28:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742316_1492 (size=12001) 2024-11-21T00:28:27,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/68cc4db6c1434d6caafcbe675e1cab14 2024-11-21T00:28:27,802 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into c97b2e23055b43c6b6e749281a4094f0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:27,802 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:27,802 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=12, startTime=1732148907686; duration=0sec 2024-11-21T00:28:27,802 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:27,802 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:27,802 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:27,804 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:27,804 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:27,804 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:27,804 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/87add02bf35f4d6888e64fdb4142172c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/8e7762731f85471da1d51c98eab2d142, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b36e9b3edc9c4ea29463db1f7ce8ba66, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/749af46beb734dfaa757838fb8d08611] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=46.9 K 2024-11-21T00:28:27,805 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 87add02bf35f4d6888e64fdb4142172c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732148905056 2024-11-21T00:28:27,806 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7762731f85471da1d51c98eab2d142, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732148905106 2024-11-21T00:28:27,806 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b36e9b3edc9c4ea29463db1f7ce8ba66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732148905521 2024-11-21T00:28:27,807 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 749af46beb734dfaa757838fb8d08611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732148906794 2024-11-21T00:28:27,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9139fa6d44714023a87cbd179127b0ce is 50, key is test_row_0/B:col10/1732148907526/Put/seqid=0 2024-11-21T00:28:27,828 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:27,831 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/cf930f2859aa4ae4b9f4b44f85db9191 is 50, key is test_row_0/C:col10/1732148906794/Put/seqid=0 2024-11-21T00:28:27,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742318_1494 (size=12139) 2024-11-21T00:28:27,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742317_1493 (size=12001) 2024-11-21T00:28:27,884 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/cf930f2859aa4ae4b9f4b44f85db9191 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/cf930f2859aa4ae4b9f4b44f85db9191 2024-11-21T00:28:27,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9139fa6d44714023a87cbd179127b0ce 2024-11-21T00:28:27,891 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into cf930f2859aa4ae4b9f4b44f85db9191(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:27,891 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:27,891 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=12, startTime=1732148907688; duration=0sec 2024-11-21T00:28:27,892 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:27,892 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:27,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/bf1b7f7f0451402fb482698f564f28ef is 50, key is test_row_0/C:col10/1732148907526/Put/seqid=0 2024-11-21T00:28:27,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148967900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742319_1495 (size=12001) 2024-11-21T00:28:27,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/bf1b7f7f0451402fb482698f564f28ef 2024-11-21T00:28:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/68cc4db6c1434d6caafcbe675e1cab14 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/68cc4db6c1434d6caafcbe675e1cab14 2024-11-21T00:28:27,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/68cc4db6c1434d6caafcbe675e1cab14, entries=150, sequenceid=87, filesize=11.7 K 2024-11-21T00:28:27,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9139fa6d44714023a87cbd179127b0ce as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9139fa6d44714023a87cbd179127b0ce 2024-11-21T00:28:27,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148967916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148967917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148967919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9139fa6d44714023a87cbd179127b0ce, entries=150, sequenceid=87, filesize=11.7 K 2024-11-21T00:28:27,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/bf1b7f7f0451402fb482698f564f28ef as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/bf1b7f7f0451402fb482698f564f28ef 2024-11-21T00:28:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:27,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/bf1b7f7f0451402fb482698f564f28ef, entries=150, sequenceid=87, filesize=11.7 K 2024-11-21T00:28:27,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5151f743f103348045956281ec3b4bb9 in 245ms, sequenceid=87, compaction requested=false 2024-11-21T00:28:27,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:28,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:28,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:28:28,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/379a1013b7bb42ada082fb2c04c14ac6 is 50, key is test_row_0/A:col10/1732148907917/Put/seqid=0 2024-11-21T00:28:28,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742320_1496 (size=14341) 2024-11-21T00:28:28,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/379a1013b7bb42ada082fb2c04c14ac6 2024-11-21T00:28:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148968065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148968073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/afd57d1e5182420fa06dc3799ed54700 is 50, key is test_row_0/B:col10/1732148907917/Put/seqid=0 2024-11-21T00:28:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148968077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148968084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148968086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742321_1497 (size=12001) 2024-11-21T00:28:28,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/afd57d1e5182420fa06dc3799ed54700 2024-11-21T00:28:28,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d0f59fc4866542adbf8ba0e2703bc6d9 is 50, key is test_row_0/C:col10/1732148907917/Put/seqid=0 2024-11-21T00:28:28,180 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a58983ae938144e1810007e1cbdb7578 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a58983ae938144e1810007e1cbdb7578 2024-11-21T00:28:28,188 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into a58983ae938144e1810007e1cbdb7578(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:28,189 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:28,189 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=12, startTime=1732148907685; duration=0sec 2024-11-21T00:28:28,189 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:28,189 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:28,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742322_1498 (size=12001) 2024-11-21T00:28:28,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148968193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148968204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148968205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148968213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148968214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,425 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:28:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148968424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148968426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148968432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148968433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148968433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d0f59fc4866542adbf8ba0e2703bc6d9 2024-11-21T00:28:28,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/379a1013b7bb42ada082fb2c04c14ac6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/379a1013b7bb42ada082fb2c04c14ac6 2024-11-21T00:28:28,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/379a1013b7bb42ada082fb2c04c14ac6, entries=200, sequenceid=114, filesize=14.0 K 2024-11-21T00:28:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/afd57d1e5182420fa06dc3799ed54700 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/afd57d1e5182420fa06dc3799ed54700 2024-11-21T00:28:28,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/afd57d1e5182420fa06dc3799ed54700, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T00:28:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d0f59fc4866542adbf8ba0e2703bc6d9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d0f59fc4866542adbf8ba0e2703bc6d9 2024-11-21T00:28:28,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d0f59fc4866542adbf8ba0e2703bc6d9, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T00:28:28,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5151f743f103348045956281ec3b4bb9 in 594ms, sequenceid=114, compaction requested=true 2024-11-21T00:28:28,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:28,641 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:28,642 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:28,642 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:28,642 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:28,642 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a58983ae938144e1810007e1cbdb7578, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/68cc4db6c1434d6caafcbe675e1cab14, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/379a1013b7bb42ada082fb2c04c14ac6] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=37.6 K 2024-11-21T00:28:28,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:28,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:28,642 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:28,644 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58983ae938144e1810007e1cbdb7578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732148906794 2024-11-21T00:28:28,645 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:28,645 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:28,645 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:28,645 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c97b2e23055b43c6b6e749281a4094f0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9139fa6d44714023a87cbd179127b0ce, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/afd57d1e5182420fa06dc3799ed54700] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=35.3 K 2024-11-21T00:28:28,645 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68cc4db6c1434d6caafcbe675e1cab14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732148907526 2024-11-21T00:28:28,646 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c97b2e23055b43c6b6e749281a4094f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732148906794 2024-11-21T00:28:28,646 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 379a1013b7bb42ada082fb2c04c14ac6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732148907912 2024-11-21T00:28:28,647 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9139fa6d44714023a87cbd179127b0ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732148907526 2024-11-21T00:28:28,647 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting afd57d1e5182420fa06dc3799ed54700, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732148907914 2024-11-21T00:28:28,681 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:28,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c1fab06f307f4d7392310f509d47a9ed is 50, key is test_row_0/B:col10/1732148907917/Put/seqid=0 2024-11-21T00:28:28,690 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:28,691 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9 is 50, key is test_row_0/A:col10/1732148907917/Put/seqid=0 2024-11-21T00:28:28,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742323_1499 (size=12241) 2024-11-21T00:28:28,748 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c1fab06f307f4d7392310f509d47a9ed as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c1fab06f307f4d7392310f509d47a9ed 2024-11-21T00:28:28,757 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into c1fab06f307f4d7392310f509d47a9ed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:28,757 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:28,757 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148908642; duration=0sec 2024-11-21T00:28:28,757 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:28,758 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:28,758 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:28,758 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:28,758 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:28,759 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:28,759 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/cf930f2859aa4ae4b9f4b44f85db9191, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/bf1b7f7f0451402fb482698f564f28ef, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d0f59fc4866542adbf8ba0e2703bc6d9] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=35.3 K 2024-11-21T00:28:28,759 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting cf930f2859aa4ae4b9f4b44f85db9191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732148906794 2024-11-21T00:28:28,759 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1b7f7f0451402fb482698f564f28ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732148907526 2024-11-21T00:28:28,760 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f59fc4866542adbf8ba0e2703bc6d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732148907914 2024-11-21T00:28:28,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742324_1500 (size=12241) 2024-11-21T00:28:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:28,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:28:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:28,772 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9 2024-11-21T00:28:28,778 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into a9cfc6d0261a4c0fb1b0ce69384ae0f9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:28,778 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:28,778 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148908641; duration=0sec 2024-11-21T00:28:28,778 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:28,778 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:28,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/633c9ed9310c4bf4be4c60f6afd6d9ed is 50, key is test_row_0/A:col10/1732148908068/Put/seqid=0 2024-11-21T00:28:28,786 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:28,786 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/570e9510d7154c20929964a90486b386 is 50, key is test_row_0/C:col10/1732148907917/Put/seqid=0 2024-11-21T00:28:28,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742325_1501 (size=16731) 2024-11-21T00:28:28,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742326_1502 (size=12241) 2024-11-21T00:28:28,828 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/570e9510d7154c20929964a90486b386 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/570e9510d7154c20929964a90486b386 2024-11-21T00:28:28,834 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into 570e9510d7154c20929964a90486b386(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:28,834 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:28,834 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148908643; duration=0sec 2024-11-21T00:28:28,834 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:28,834 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:28,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148968860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148968863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148968864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148968885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:28,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148968887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148968992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148968992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148968992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148969013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148969013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T00:28:29,223 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:29,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148969212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:29,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/633c9ed9310c4bf4be4c60f6afd6d9ed 2024-11-21T00:28:29,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148969212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148969216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-21T00:28:29,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:29,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:29,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:28:29,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148969232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/b9ebbb6982244c569c1aebd0d257cc9c is 50, key is test_row_0/B:col10/1732148908068/Put/seqid=0 2024-11-21T00:28:29,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148969242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742327_1503 (size=12051) 2024-11-21T00:28:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:28:29,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:28:29,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:29,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:28:29,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:29,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148969532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148969532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148969533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:28:29,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148969552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148969562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:29,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:28:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/b9ebbb6982244c569c1aebd0d257cc9c 2024-11-21T00:28:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:29,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/130906f046224cf092a7ddf7f3744c65 is 50, key is test_row_0/C:col10/1732148908068/Put/seqid=0 2024-11-21T00:28:29,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742328_1504 (size=12051) 2024-11-21T00:28:29,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/130906f046224cf092a7ddf7f3744c65 2024-11-21T00:28:29,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/633c9ed9310c4bf4be4c60f6afd6d9ed as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/633c9ed9310c4bf4be4c60f6afd6d9ed 2024-11-21T00:28:29,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/633c9ed9310c4bf4be4c60f6afd6d9ed, entries=250, sequenceid=129, filesize=16.3 K 2024-11-21T00:28:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/b9ebbb6982244c569c1aebd0d257cc9c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b9ebbb6982244c569c1aebd0d257cc9c 2024-11-21T00:28:29,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b9ebbb6982244c569c1aebd0d257cc9c, entries=150, sequenceid=129, filesize=11.8 K 2024-11-21T00:28:29,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/130906f046224cf092a7ddf7f3744c65 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/130906f046224cf092a7ddf7f3744c65 2024-11-21T00:28:29,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/130906f046224cf092a7ddf7f3744c65, entries=150, sequenceid=129, filesize=11.8 K 2024-11-21T00:28:29,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5151f743f103348045956281ec3b4bb9 in 1039ms, sequenceid=129, compaction requested=false 2024-11-21T00:28:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:29,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T00:28:29,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:29,852 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T00:28:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:28:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e635976b175a4ea29225e1f18d707040 is 50, key is test_row_0/A:col10/1732148908859/Put/seqid=0 2024-11-21T00:28:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742329_1505 (size=12151) 2024-11-21T00:28:29,903 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e635976b175a4ea29225e1f18d707040 2024-11-21T00:28:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/659841b7c4674892bec338160b4258b4 is 50, key is test_row_0/B:col10/1732148908859/Put/seqid=0 2024-11-21T00:28:29,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742330_1506 (size=12151) 2024-11-21T00:28:30,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:30,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148970107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148970115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148970124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148970125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148970125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148970232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148970248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148970248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148970249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148970249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:28:30,363 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/659841b7c4674892bec338160b4258b4 2024-11-21T00:28:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1c7d13c8fcd64ae79bfef074800007d7 is 50, key is test_row_0/C:col10/1732148908859/Put/seqid=0 2024-11-21T00:28:30,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742331_1507 (size=12151) 2024-11-21T00:28:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148970452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148970483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148970483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148970483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148970483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148970762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,804 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1c7d13c8fcd64ae79bfef074800007d7 2024-11-21T00:28:30,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148970792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148970793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148970793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148970793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e635976b175a4ea29225e1f18d707040 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e635976b175a4ea29225e1f18d707040 2024-11-21T00:28:30,840 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e635976b175a4ea29225e1f18d707040, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T00:28:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/659841b7c4674892bec338160b4258b4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/659841b7c4674892bec338160b4258b4 2024-11-21T00:28:30,851 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/659841b7c4674892bec338160b4258b4, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T00:28:30,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1c7d13c8fcd64ae79bfef074800007d7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1c7d13c8fcd64ae79bfef074800007d7 2024-11-21T00:28:30,860 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1c7d13c8fcd64ae79bfef074800007d7, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T00:28:30,861 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5151f743f103348045956281ec3b4bb9 in 1009ms, sequenceid=154, compaction requested=true 2024-11-21T00:28:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-21T00:28:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-21T00:28:30,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T00:28:30,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6650 sec 2024-11-21T00:28:30,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.6730 sec 2024-11-21T00:28:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:31,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T00:28:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:31,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/3181e0b310a84e1eb70149b09a678621 is 50, key is test_row_0/A:col10/1732148911283/Put/seqid=0 2024-11-21T00:28:31,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742332_1508 (size=12147) 2024-11-21T00:28:31,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/3181e0b310a84e1eb70149b09a678621 2024-11-21T00:28:31,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/a67761ac84a54727bb78c51f96709b64 is 50, key is test_row_0/B:col10/1732148911283/Put/seqid=0 2024-11-21T00:28:31,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742333_1509 (size=9757) 2024-11-21T00:28:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T00:28:31,372 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:31,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-21T00:28:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:28:31,374 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:31,375 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:31,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:31,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148971392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148971409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148971409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148971410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148971412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:28:31,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:28:31,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:31,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148971522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148971532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148971532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148971536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148971534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:28:31,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:31,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:28:31,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148971742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/a67761ac84a54727bb78c51f96709b64 2024-11-21T00:28:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148971752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148971754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148971754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148971753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:31,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/3e67ddd332f347d8adc8e41a1837556a is 50, key is test_row_0/C:col10/1732148911283/Put/seqid=0 2024-11-21T00:28:31,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:28:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742334_1510 (size=9757) 2024-11-21T00:28:31,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:28:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:31,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:28:32,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148972066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148972085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148972085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148972086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148972092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:28:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:32,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/3e67ddd332f347d8adc8e41a1837556a 2024-11-21T00:28:32,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/3181e0b310a84e1eb70149b09a678621 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3181e0b310a84e1eb70149b09a678621 2024-11-21T00:28:32,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3181e0b310a84e1eb70149b09a678621, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T00:28:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/a67761ac84a54727bb78c51f96709b64 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/a67761ac84a54727bb78c51f96709b64 2024-11-21T00:28:32,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/a67761ac84a54727bb78c51f96709b64, entries=100, sequenceid=169, filesize=9.5 K 2024-11-21T00:28:32,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/3e67ddd332f347d8adc8e41a1837556a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/3e67ddd332f347d8adc8e41a1837556a 2024-11-21T00:28:32,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/3e67ddd332f347d8adc8e41a1837556a, entries=100, sequenceid=169, filesize=9.5 K 2024-11-21T00:28:32,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 5151f743f103348045956281ec3b4bb9 in 1014ms, sequenceid=169, compaction requested=true 2024-11-21T00:28:32,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:32,299 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:32,300 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:32,301 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53270 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:32,301 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:32,301 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,301 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/633c9ed9310c4bf4be4c60f6afd6d9ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e635976b175a4ea29225e1f18d707040, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3181e0b310a84e1eb70149b09a678621] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=52.0 K 2024-11-21T00:28:32,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T00:28:32,304 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:32,304 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:32,304 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,304 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c1fab06f307f4d7392310f509d47a9ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b9ebbb6982244c569c1aebd0d257cc9c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/659841b7c4674892bec338160b4258b4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/a67761ac84a54727bb78c51f96709b64] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=45.1 K 2024-11-21T00:28:32,305 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9cfc6d0261a4c0fb1b0ce69384ae0f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732148907914 2024-11-21T00:28:32,305 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c1fab06f307f4d7392310f509d47a9ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732148907914 2024-11-21T00:28:32,306 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633c9ed9310c4bf4be4c60f6afd6d9ed, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732148908068 2024-11-21T00:28:32,311 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ebbb6982244c569c1aebd0d257cc9c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732148908068 2024-11-21T00:28:32,311 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 659841b7c4674892bec338160b4258b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732148908859 2024-11-21T00:28:32,312 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a67761ac84a54727bb78c51f96709b64, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148910117 2024-11-21T00:28:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,312 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e635976b175a4ea29225e1f18d707040, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732148908859 2024-11-21T00:28:32,312 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T00:28:32,313 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3181e0b310a84e1eb70149b09a678621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148910117 2024-11-21T00:28:32,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/1a4c48964ce7488bb2988440bdaf1f6b is 50, key is test_row_0/A:col10/1732148911390/Put/seqid=0 2024-11-21T00:28:32,357 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:32,358 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9ba3570c90674e4f91034420d5f0e18a is 50, key is test_row_0/B:col10/1732148911283/Put/seqid=0 2024-11-21T00:28:32,367 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:32,367 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/0fdbe3c1c6934353bf9e1004c7186077 is 50, key is test_row_0/A:col10/1732148911283/Put/seqid=0 2024-11-21T00:28:32,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742335_1511 (size=12151) 2024-11-21T00:28:32,423 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/1a4c48964ce7488bb2988440bdaf1f6b 2024-11-21T00:28:32,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742336_1512 (size=12527) 2024-11-21T00:28:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742337_1513 (size=12527) 2024-11-21T00:28:32,463 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/0fdbe3c1c6934353bf9e1004c7186077 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/0fdbe3c1c6934353bf9e1004c7186077 2024-11-21T00:28:32,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/014919fc5a2b439e98b4e2b85222f061 is 50, key is test_row_0/B:col10/1732148911390/Put/seqid=0 2024-11-21T00:28:32,473 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into 0fdbe3c1c6934353bf9e1004c7186077(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:32,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:32,473 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=12, startTime=1732148912299; duration=0sec 2024-11-21T00:28:32,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:32,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:32,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:32,474 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:32,474 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:32,474 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,474 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/570e9510d7154c20929964a90486b386, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/130906f046224cf092a7ddf7f3744c65, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1c7d13c8fcd64ae79bfef074800007d7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/3e67ddd332f347d8adc8e41a1837556a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=45.1 K 2024-11-21T00:28:32,474 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 570e9510d7154c20929964a90486b386, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732148907914 2024-11-21T00:28:32,474 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130906f046224cf092a7ddf7f3744c65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732148908068 2024-11-21T00:28:32,475 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c7d13c8fcd64ae79bfef074800007d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732148908859 2024-11-21T00:28:32,475 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e67ddd332f347d8adc8e41a1837556a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148910117 2024-11-21T00:28:32,492 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:32,492 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/576b88eceeca4c25baa8b2ba9f722957 is 50, key is test_row_0/C:col10/1732148911283/Put/seqid=0 2024-11-21T00:28:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:28:32,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742338_1514 (size=12151) 2024-11-21T00:28:32,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742339_1515 (size=12527) 2024-11-21T00:28:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:32,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148972625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148972627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148972633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148972636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148972634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148972742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148972744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148972766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148972767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148972768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,842 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9ba3570c90674e4f91034420d5f0e18a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9ba3570c90674e4f91034420d5f0e18a 2024-11-21T00:28:32,849 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 9ba3570c90674e4f91034420d5f0e18a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:32,849 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:32,849 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=12, startTime=1732148912300; duration=0sec 2024-11-21T00:28:32,849 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:32,849 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:32,921 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/014919fc5a2b439e98b4e2b85222f061 2024-11-21T00:28:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/e09a6d5009e74dc9ad1b266bd3830fd6 is 50, key is test_row_0/C:col10/1732148911390/Put/seqid=0 2024-11-21T00:28:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742340_1516 (size=12151) 2024-11-21T00:28:32,936 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/e09a6d5009e74dc9ad1b266bd3830fd6 2024-11-21T00:28:32,942 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/576b88eceeca4c25baa8b2ba9f722957 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/576b88eceeca4c25baa8b2ba9f722957 2024-11-21T00:28:32,947 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into 576b88eceeca4c25baa8b2ba9f722957(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:32,947 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:32,947 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=12, startTime=1732148912300; duration=0sec 2024-11-21T00:28:32,947 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:32,947 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:32,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/1a4c48964ce7488bb2988440bdaf1f6b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1a4c48964ce7488bb2988440bdaf1f6b 2024-11-21T00:28:32,951 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1a4c48964ce7488bb2988440bdaf1f6b, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T00:28:32,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/014919fc5a2b439e98b4e2b85222f061 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/014919fc5a2b439e98b4e2b85222f061 2024-11-21T00:28:32,957 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/014919fc5a2b439e98b4e2b85222f061, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T00:28:32,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/e09a6d5009e74dc9ad1b266bd3830fd6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e09a6d5009e74dc9ad1b266bd3830fd6 2024-11-21T00:28:32,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148972962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148972964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:32,972 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e09a6d5009e74dc9ad1b266bd3830fd6, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T00:28:32,975 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5151f743f103348045956281ec3b4bb9 in 662ms, sequenceid=191, compaction requested=false 2024-11-21T00:28:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-21T00:28:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-21T00:28:32,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T00:28:32,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-21T00:28:32,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.6070 sec 2024-11-21T00:28:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:32,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T00:28:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:32,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/b7586015d070436cad233a6466385e2d is 50, key is test_row_0/A:col10/1732148912985/Put/seqid=0 2024-11-21T00:28:33,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742341_1517 (size=16931) 2024-11-21T00:28:33,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/b7586015d070436cad233a6466385e2d 2024-11-21T00:28:33,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/ef4f086726b1455d870def7bc7f98167 is 50, key is test_row_0/B:col10/1732148912985/Put/seqid=0 2024-11-21T00:28:33,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742342_1518 (size=12151) 2024-11-21T00:28:33,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148973087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148973088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148973100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148973213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148973215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148973224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148973274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148973282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148973424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148973432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148973444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/ef4f086726b1455d870def7bc7f98167 2024-11-21T00:28:33,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1fb0ea7c604f4c2abe5b795893084943 is 50, key is test_row_0/C:col10/1732148912985/Put/seqid=0 2024-11-21T00:28:33,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742343_1519 (size=12151) 2024-11-21T00:28:33,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1fb0ea7c604f4c2abe5b795893084943 2024-11-21T00:28:33,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/b7586015d070436cad233a6466385e2d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/b7586015d070436cad233a6466385e2d 2024-11-21T00:28:33,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/b7586015d070436cad233a6466385e2d, entries=250, sequenceid=210, filesize=16.5 K 2024-11-21T00:28:33,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/ef4f086726b1455d870def7bc7f98167 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ef4f086726b1455d870def7bc7f98167 2024-11-21T00:28:33,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ef4f086726b1455d870def7bc7f98167, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T00:28:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1fb0ea7c604f4c2abe5b795893084943 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1fb0ea7c604f4c2abe5b795893084943 2024-11-21T00:28:33,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1fb0ea7c604f4c2abe5b795893084943, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T00:28:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T00:28:33,522 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:33,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5151f743f103348045956281ec3b4bb9 in 535ms, sequenceid=210, compaction requested=true 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:33,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:33,522 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:33,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:33,523 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:33,523 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:33,523 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:33,523 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/0fdbe3c1c6934353bf9e1004c7186077, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1a4c48964ce7488bb2988440bdaf1f6b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/b7586015d070436cad233a6466385e2d] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=40.6 K 2024-11-21T00:28:33,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:33,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:33,524 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:33,524 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9ba3570c90674e4f91034420d5f0e18a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/014919fc5a2b439e98b4e2b85222f061, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ef4f086726b1455d870def7bc7f98167] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.0 K 2024-11-21T00:28:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-21T00:28:33,526 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba3570c90674e4f91034420d5f0e18a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148908883 2024-11-21T00:28:33,526 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fdbe3c1c6934353bf9e1004c7186077, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148908883 2024-11-21T00:28:33,526 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 014919fc5a2b439e98b4e2b85222f061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732148911390 2024-11-21T00:28:33,526 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4c48964ce7488bb2988440bdaf1f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732148911390 2024-11-21T00:28:33,526 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ef4f086726b1455d870def7bc7f98167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148912634 2024-11-21T00:28:33,527 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7586015d070436cad233a6466385e2d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148912612 2024-11-21T00:28:33,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:28:33,529 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:33,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:33,553 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:33,553 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/71764c05aea14d9cb8c13fbe670fd8bf is 50, key is test_row_0/A:col10/1732148912985/Put/seqid=0 2024-11-21T00:28:33,562 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:33,563 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/273c2afe2e4341089a2ac7eab77cafd8 is 50, key is test_row_0/B:col10/1732148912985/Put/seqid=0 2024-11-21T00:28:33,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742344_1520 (size=12629) 2024-11-21T00:28:33,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742345_1521 (size=12629) 2024-11-21T00:28:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:28:33,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T00:28:33,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:33,682 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T00:28:33,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/438b2b50eeec440b86d042442bf6c29c is 50, key is test_row_0/A:col10/1732148913086/Put/seqid=0 2024-11-21T00:28:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742346_1522 (size=12151) 2024-11-21T00:28:33,734 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/438b2b50eeec440b86d042442bf6c29c 2024-11-21T00:28:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/10d479665f5542a2b4a8e97ee9970e30 is 50, key is test_row_0/B:col10/1732148913086/Put/seqid=0 2024-11-21T00:28:33,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742347_1523 (size=12151) 2024-11-21T00:28:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:33,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:33,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148973803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148973804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148973806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148973810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148973811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:28:33,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148973925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148973925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148973932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148973932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:33,999 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/71764c05aea14d9cb8c13fbe670fd8bf as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/71764c05aea14d9cb8c13fbe670fd8bf 2024-11-21T00:28:34,009 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into 71764c05aea14d9cb8c13fbe670fd8bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:34,009 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:34,009 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148913522; duration=0sec 2024-11-21T00:28:34,009 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:34,009 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:34,009 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:34,010 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:34,010 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:34,010 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:34,010 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/576b88eceeca4c25baa8b2ba9f722957, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e09a6d5009e74dc9ad1b266bd3830fd6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1fb0ea7c604f4c2abe5b795893084943] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.0 K 2024-11-21T00:28:34,011 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 576b88eceeca4c25baa8b2ba9f722957, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732148908883 2024-11-21T00:28:34,011 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09a6d5009e74dc9ad1b266bd3830fd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732148911390 2024-11-21T00:28:34,011 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb0ea7c604f4c2abe5b795893084943, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148912634 2024-11-21T00:28:34,026 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:34,027 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/fe03b7e133c74270ab40dea8fd2ab305 is 50, key is test_row_0/C:col10/1732148912985/Put/seqid=0 2024-11-21T00:28:34,031 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/273c2afe2e4341089a2ac7eab77cafd8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/273c2afe2e4341089a2ac7eab77cafd8 2024-11-21T00:28:34,041 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 273c2afe2e4341089a2ac7eab77cafd8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:34,041 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:34,041 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148913522; duration=0sec 2024-11-21T00:28:34,041 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:34,041 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:34,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742348_1524 (size=12629) 2024-11-21T00:28:34,064 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/fe03b7e133c74270ab40dea8fd2ab305 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/fe03b7e133c74270ab40dea8fd2ab305 2024-11-21T00:28:34,076 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into fe03b7e133c74270ab40dea8fd2ab305(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:34,076 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:34,076 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148913522; duration=0sec 2024-11-21T00:28:34,076 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:34,076 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:34,153 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/10d479665f5542a2b4a8e97ee9970e30 2024-11-21T00:28:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:28:34,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148974153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148974153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148974165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148974166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/e6f05fc9f6de4174b9ced9d8f765b80e is 50, key is test_row_0/C:col10/1732148913086/Put/seqid=0 2024-11-21T00:28:34,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742349_1525 (size=12151) 2024-11-21T00:28:34,236 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/e6f05fc9f6de4174b9ced9d8f765b80e 2024-11-21T00:28:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/438b2b50eeec440b86d042442bf6c29c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/438b2b50eeec440b86d042442bf6c29c 2024-11-21T00:28:34,253 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/438b2b50eeec440b86d042442bf6c29c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-21T00:28:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/10d479665f5542a2b4a8e97ee9970e30 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/10d479665f5542a2b4a8e97ee9970e30 2024-11-21T00:28:34,262 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/10d479665f5542a2b4a8e97ee9970e30, entries=150, sequenceid=230, filesize=11.9 K 2024-11-21T00:28:34,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/e6f05fc9f6de4174b9ced9d8f765b80e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e6f05fc9f6de4174b9ced9d8f765b80e 2024-11-21T00:28:34,269 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e6f05fc9f6de4174b9ced9d8f765b80e, entries=150, sequenceid=230, filesize=11.9 K 2024-11-21T00:28:34,271 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5151f743f103348045956281ec3b4bb9 in 588ms, sequenceid=230, compaction requested=false 2024-11-21T00:28:34,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:34,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:34,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-21T00:28:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-21T00:28:34,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T00:28:34,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 743 msec 2024-11-21T00:28:34,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 761 msec 2024-11-21T00:28:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:34,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:28:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:34,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/6c0c4fcc4bc94b4f9a4de846325b2400 is 50, key is test_row_0/A:col10/1732148913785/Put/seqid=0 2024-11-21T00:28:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742350_1526 (size=14541) 2024-11-21T00:28:34,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/6c0c4fcc4bc94b4f9a4de846325b2400 2024-11-21T00:28:34,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/f207c279c89049f68bab8fb191dd149b is 50, key is test_row_0/B:col10/1732148913785/Put/seqid=0 2024-11-21T00:28:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742351_1527 (size=12151) 2024-11-21T00:28:34,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148974559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148974571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148974572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148974577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T00:28:34,662 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:34,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-21T00:28:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:28:34,669 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:34,670 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:34,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:34,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148974692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148974714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148974714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148974714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:28:34,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:28:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:34,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:34,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148974822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,829 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148974908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148974936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148974936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148974942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:34,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/f207c279c89049f68bab8fb191dd149b 2024-11-21T00:28:34,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:28:34,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:34,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:34,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:34,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:34,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:28:35,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d1e0c15c64e345b1a9274d27e5e0b19d is 50, key is test_row_0/C:col10/1732148913785/Put/seqid=0 2024-11-21T00:28:35,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742352_1528 (size=12151) 2024-11-21T00:28:35,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d1e0c15c64e345b1a9274d27e5e0b19d 2024-11-21T00:28:35,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/6c0c4fcc4bc94b4f9a4de846325b2400 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6c0c4fcc4bc94b4f9a4de846325b2400 2024-11-21T00:28:35,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6c0c4fcc4bc94b4f9a4de846325b2400, entries=200, sequenceid=250, filesize=14.2 K 2024-11-21T00:28:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/f207c279c89049f68bab8fb191dd149b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f207c279c89049f68bab8fb191dd149b 2024-11-21T00:28:35,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f207c279c89049f68bab8fb191dd149b, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T00:28:35,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d1e0c15c64e345b1a9274d27e5e0b19d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d1e0c15c64e345b1a9274d27e5e0b19d 2024-11-21T00:28:35,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d1e0c15c64e345b1a9274d27e5e0b19d, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T00:28:35,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5151f743f103348045956281ec3b4bb9 in 587ms, sequenceid=250, compaction requested=true 2024-11-21T00:28:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:35,063 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:35,063 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:35,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:35,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:35,064 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:35,064 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:35,064 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:35,064 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:35,064 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:35,064 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/71764c05aea14d9cb8c13fbe670fd8bf, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/438b2b50eeec440b86d042442bf6c29c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6c0c4fcc4bc94b4f9a4de846325b2400] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=38.4 K 2024-11-21T00:28:35,064 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:35,064 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/273c2afe2e4341089a2ac7eab77cafd8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/10d479665f5542a2b4a8e97ee9970e30, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f207c279c89049f68bab8fb191dd149b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.1 K 2024-11-21T00:28:35,065 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 71764c05aea14d9cb8c13fbe670fd8bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148912634 2024-11-21T00:28:35,065 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 273c2afe2e4341089a2ac7eab77cafd8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148912634 2024-11-21T00:28:35,065 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d479665f5542a2b4a8e97ee9970e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732148913085 2024-11-21T00:28:35,065 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 438b2b50eeec440b86d042442bf6c29c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732148913085 2024-11-21T00:28:35,065 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0c4fcc4bc94b4f9a4de846325b2400, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148913785 2024-11-21T00:28:35,065 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f207c279c89049f68bab8fb191dd149b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148913785 2024-11-21T00:28:35,083 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:35,083 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/33ca0b79d3f4431e98950601f6b7d4da is 50, key is test_row_0/B:col10/1732148913785/Put/seqid=0 2024-11-21T00:28:35,102 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:35,102 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/6864aa62af8a4f548f4df97ef3fd8cc0 is 50, key is test_row_0/A:col10/1732148913785/Put/seqid=0 2024-11-21T00:28:35,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T00:28:35,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:35,134 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T00:28:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/29b24d12eaca400fbf49be7980b76062 is 50, key is test_row_0/A:col10/1732148914568/Put/seqid=0 2024-11-21T00:28:35,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742354_1530 (size=12731) 2024-11-21T00:28:35,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742353_1529 (size=12731) 2024-11-21T00:28:35,216 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/33ca0b79d3f4431e98950601f6b7d4da as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/33ca0b79d3f4431e98950601f6b7d4da 2024-11-21T00:28:35,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742355_1531 (size=12301) 2024-11-21T00:28:35,229 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 33ca0b79d3f4431e98950601f6b7d4da(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:35,229 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:35,229 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148915063; duration=0sec 2024-11-21T00:28:35,229 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:35,229 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:35,229 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:35,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:35,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:35,232 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:35,232 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/fe03b7e133c74270ab40dea8fd2ab305, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e6f05fc9f6de4174b9ced9d8f765b80e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d1e0c15c64e345b1a9274d27e5e0b19d] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.1 K 2024-11-21T00:28:35,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe03b7e133c74270ab40dea8fd2ab305, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732148912634 2024-11-21T00:28:35,232 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f05fc9f6de4174b9ced9d8f765b80e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732148913085 2024-11-21T00:28:35,233 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e0c15c64e345b1a9274d27e5e0b19d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148913785 2024-11-21T00:28:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:35,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:35,243 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:35,243 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/00f1decf22244231ab032261a42249c8 is 50, key is test_row_0/C:col10/1732148913785/Put/seqid=0 2024-11-21T00:28:35,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742356_1532 (size=12731) 2024-11-21T00:28:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:28:35,299 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/00f1decf22244231ab032261a42249c8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/00f1decf22244231ab032261a42249c8 2024-11-21T00:28:35,303 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into 00f1decf22244231ab032261a42249c8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:35,303 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:35,303 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148915064; duration=0sec 2024-11-21T00:28:35,304 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:35,304 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148975291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148975299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148975300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148975312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148975422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148975422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148975422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148975432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,585 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/6864aa62af8a4f548f4df97ef3fd8cc0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6864aa62af8a4f548f4df97ef3fd8cc0 2024-11-21T00:28:35,593 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into 6864aa62af8a4f548f4df97ef3fd8cc0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:35,593 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:35,593 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148915063; duration=0sec 2024-11-21T00:28:35,593 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:35,594 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:35,622 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/29b24d12eaca400fbf49be7980b76062 2024-11-21T00:28:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/21109253651d492f89cf86f5a73c085c is 50, key is test_row_0/B:col10/1732148914568/Put/seqid=0 2024-11-21T00:28:35,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148975633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148975642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148975642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742357_1533 (size=12301) 2024-11-21T00:28:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148975653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:28:35,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148975955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148975955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148975955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148975962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:36,051 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/21109253651d492f89cf86f5a73c085c 2024-11-21T00:28:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ece4cb96e3a14957b80599f7d052878d is 50, key is test_row_0/C:col10/1732148914568/Put/seqid=0 2024-11-21T00:28:36,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742358_1534 (size=12301) 2024-11-21T00:28:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148976482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148976482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148976482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148976482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:36,518 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ece4cb96e3a14957b80599f7d052878d 2024-11-21T00:28:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/29b24d12eaca400fbf49be7980b76062 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/29b24d12eaca400fbf49be7980b76062 2024-11-21T00:28:36,529 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/29b24d12eaca400fbf49be7980b76062, entries=150, sequenceid=270, filesize=12.0 K 2024-11-21T00:28:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/21109253651d492f89cf86f5a73c085c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/21109253651d492f89cf86f5a73c085c 2024-11-21T00:28:36,566 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/21109253651d492f89cf86f5a73c085c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-21T00:28:36,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ece4cb96e3a14957b80599f7d052878d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ece4cb96e3a14957b80599f7d052878d 2024-11-21T00:28:36,582 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ece4cb96e3a14957b80599f7d052878d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-21T00:28:36,583 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5151f743f103348045956281ec3b4bb9 in 1449ms, sequenceid=270, compaction requested=false 2024-11-21T00:28:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-21T00:28:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-21T00:28:36,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T00:28:36,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-21T00:28:36,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9260 sec 2024-11-21T00:28:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T00:28:36,817 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:36,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T00:28:36,820 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:36,822 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:36,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:28:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:36,857 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:36,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:28:36,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/61b61c1dd9f749c6979c034a92783de1 is 50, key is test_row_0/A:col10/1732148915286/Put/seqid=0 2024-11-21T00:28:36,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742359_1535 (size=14741) 2024-11-21T00:28:36,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/61b61c1dd9f749c6979c034a92783de1 2024-11-21T00:28:36,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/12ff07f290fe4443a966f4a97e05e23a is 50, key is test_row_0/B:col10/1732148915286/Put/seqid=0 2024-11-21T00:28:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:28:36,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742360_1536 (size=12301) 2024-11-21T00:28:36,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:36,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:36,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148977121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:28:37,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:37,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148977252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:37,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:37,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/12ff07f290fe4443a966f4a97e05e23a 2024-11-21T00:28:37,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/988697bbc5b34252a48882fd2581af14 is 50, key is test_row_0/C:col10/1732148915286/Put/seqid=0 2024-11-21T00:28:37,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742361_1537 (size=12301) 2024-11-21T00:28:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:28:37,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148977477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148977502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148977502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148977502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148977502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,518 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:37,518 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,518 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,524 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148977792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:37,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/988697bbc5b34252a48882fd2581af14 2024-11-21T00:28:37,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/61b61c1dd9f749c6979c034a92783de1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/61b61c1dd9f749c6979c034a92783de1 2024-11-21T00:28:37,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/61b61c1dd9f749c6979c034a92783de1, entries=200, sequenceid=290, filesize=14.4 K 2024-11-21T00:28:37,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/12ff07f290fe4443a966f4a97e05e23a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/12ff07f290fe4443a966f4a97e05e23a 2024-11-21T00:28:37,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/12ff07f290fe4443a966f4a97e05e23a, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:28:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/988697bbc5b34252a48882fd2581af14 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/988697bbc5b34252a48882fd2581af14 2024-11-21T00:28:37,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/988697bbc5b34252a48882fd2581af14, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T00:28:37,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5151f743f103348045956281ec3b4bb9 in 1006ms, sequenceid=290, compaction requested=true 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:37,863 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:37,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:28:37,863 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:37,869 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:37,869 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:37,869 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,869 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/33ca0b79d3f4431e98950601f6b7d4da, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/21109253651d492f89cf86f5a73c085c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/12ff07f290fe4443a966f4a97e05e23a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.5 K 2024-11-21T00:28:37,870 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:37,870 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:37,870 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,870 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6864aa62af8a4f548f4df97ef3fd8cc0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/29b24d12eaca400fbf49be7980b76062, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/61b61c1dd9f749c6979c034a92783de1] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=38.8 K 2024-11-21T00:28:37,870 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6864aa62af8a4f548f4df97ef3fd8cc0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148913785 2024-11-21T00:28:37,870 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ca0b79d3f4431e98950601f6b7d4da, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148913785 2024-11-21T00:28:37,870 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b24d12eaca400fbf49be7980b76062, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732148914532 2024-11-21T00:28:37,871 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 21109253651d492f89cf86f5a73c085c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732148914532 2024-11-21T00:28:37,871 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61b61c1dd9f749c6979c034a92783de1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148915286 2024-11-21T00:28:37,871 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 12ff07f290fe4443a966f4a97e05e23a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148915286 2024-11-21T00:28:37,882 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:37,883 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e79d7d1a4c524160becbd2e37b4a3e1b is 50, key is test_row_0/A:col10/1732148915286/Put/seqid=0 2024-11-21T00:28:37,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T00:28:37,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:37,916 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:37,918 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:37,918 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/31a37b4b1a4b49b2a614b0d2673eac22 is 50, key is test_row_0/B:col10/1732148915286/Put/seqid=0 2024-11-21T00:28:37,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a88f0f4b45a547c7bcd742b8c51ec35d is 50, key is test_row_0/A:col10/1732148917002/Put/seqid=0 2024-11-21T00:28:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:28:37,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742362_1538 (size=12983) 2024-11-21T00:28:38,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742363_1539 (size=12983) 2024-11-21T00:28:38,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742364_1540 (size=12301) 2024-11-21T00:28:38,036 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a88f0f4b45a547c7bcd742b8c51ec35d 2024-11-21T00:28:38,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/2fa66121d7bc4479b553e4519f586ce8 is 50, key is test_row_0/B:col10/1732148917002/Put/seqid=0 2024-11-21T00:28:38,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742365_1541 (size=12301) 2024-11-21T00:28:38,133 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/2fa66121d7bc4479b553e4519f586ce8 2024-11-21T00:28:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/824b82b720ff462e82428f301f9dd9cc is 50, key is test_row_0/C:col10/1732148917002/Put/seqid=0 2024-11-21T00:28:38,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742366_1542 (size=12301) 2024-11-21T00:28:38,213 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/824b82b720ff462e82428f301f9dd9cc 2024-11-21T00:28:38,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/a88f0f4b45a547c7bcd742b8c51ec35d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a88f0f4b45a547c7bcd742b8c51ec35d 2024-11-21T00:28:38,233 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a88f0f4b45a547c7bcd742b8c51ec35d, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T00:28:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/2fa66121d7bc4479b553e4519f586ce8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/2fa66121d7bc4479b553e4519f586ce8 2024-11-21T00:28:38,240 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/2fa66121d7bc4479b553e4519f586ce8, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T00:28:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/824b82b720ff462e82428f301f9dd9cc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/824b82b720ff462e82428f301f9dd9cc 2024-11-21T00:28:38,248 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/824b82b720ff462e82428f301f9dd9cc, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T00:28:38,249 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 5151f743f103348045956281ec3b4bb9 in 333ms, sequenceid=309, compaction requested=true 2024-11-21T00:28:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T00:28:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T00:28:38,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T00:28:38,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-21T00:28:38,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4370 sec 2024-11-21T00:28:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:38,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:38,411 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e79d7d1a4c524160becbd2e37b4a3e1b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e79d7d1a4c524160becbd2e37b4a3e1b 2024-11-21T00:28:38,421 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/31a37b4b1a4b49b2a614b0d2673eac22 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/31a37b4b1a4b49b2a614b0d2673eac22 2024-11-21T00:28:38,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e3a1b8f86ea34fdba424ceeee32777bc is 50, key is test_row_0/A:col10/1732148918366/Put/seqid=0 2024-11-21T00:28:38,426 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into e79d7d1a4c524160becbd2e37b4a3e1b(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:38,426 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,426 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148917863; duration=0sec 2024-11-21T00:28:38,426 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:38,426 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:38,426 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:38,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:38,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:38,436 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:38,436 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/00f1decf22244231ab032261a42249c8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ece4cb96e3a14957b80599f7d052878d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/988697bbc5b34252a48882fd2581af14, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/824b82b720ff462e82428f301f9dd9cc] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=48.5 K 2024-11-21T00:28:38,437 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f1decf22244231ab032261a42249c8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148913785 2024-11-21T00:28:38,437 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece4cb96e3a14957b80599f7d052878d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732148914532 2024-11-21T00:28:38,437 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988697bbc5b34252a48882fd2581af14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148915286 2024-11-21T00:28:38,437 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824b82b720ff462e82428f301f9dd9cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148917002 2024-11-21T00:28:38,443 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 31a37b4b1a4b49b2a614b0d2673eac22(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:38,443 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,443 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148917863; duration=0sec 2024-11-21T00:28:38,443 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:38,443 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:38,458 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:38,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742367_1543 (size=12301) 2024-11-21T00:28:38,459 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/40f1fa75943749bf88651690867e7873 is 50, key is test_row_0/C:col10/1732148917002/Put/seqid=0 2024-11-21T00:28:38,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e3a1b8f86ea34fdba424ceeee32777bc 2024-11-21T00:28:38,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/ec89fe03a0754b71a63f493b16cfffd5 is 50, key is test_row_0/B:col10/1732148918366/Put/seqid=0 2024-11-21T00:28:38,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742368_1544 (size=13017) 2024-11-21T00:28:38,486 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/40f1fa75943749bf88651690867e7873 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/40f1fa75943749bf88651690867e7873 2024-11-21T00:28:38,491 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into 40f1fa75943749bf88651690867e7873(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:38,491 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,491 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=12, startTime=1732148917863; duration=0sec 2024-11-21T00:28:38,491 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:38,491 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:38,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742369_1545 (size=12301) 2024-11-21T00:28:38,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/ec89fe03a0754b71a63f493b16cfffd5 2024-11-21T00:28:38,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d25491bbf8ac4fab9fc7fe871c558d20 is 50, key is test_row_0/C:col10/1732148918366/Put/seqid=0 2024-11-21T00:28:38,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742370_1546 (size=12301) 2024-11-21T00:28:38,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d25491bbf8ac4fab9fc7fe871c558d20 2024-11-21T00:28:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/e3a1b8f86ea34fdba424ceeee32777bc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e3a1b8f86ea34fdba424ceeee32777bc 2024-11-21T00:28:38,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e3a1b8f86ea34fdba424ceeee32777bc, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T00:28:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/ec89fe03a0754b71a63f493b16cfffd5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ec89fe03a0754b71a63f493b16cfffd5 2024-11-21T00:28:38,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ec89fe03a0754b71a63f493b16cfffd5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T00:28:38,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/d25491bbf8ac4fab9fc7fe871c558d20 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d25491bbf8ac4fab9fc7fe871c558d20 2024-11-21T00:28:38,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d25491bbf8ac4fab9fc7fe871c558d20, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T00:28:38,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=87.22 KB/89310 for 5151f743f103348045956281ec3b4bb9 in 211ms, sequenceid=320, compaction requested=true 2024-11-21T00:28:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:38,604 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:28:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:38,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:28:38,605 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:38,605 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:38,605 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:38,605 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e79d7d1a4c524160becbd2e37b4a3e1b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a88f0f4b45a547c7bcd742b8c51ec35d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e3a1b8f86ea34fdba424ceeee32777bc] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.7 K 2024-11-21T00:28:38,605 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e79d7d1a4c524160becbd2e37b4a3e1b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148915286 2024-11-21T00:28:38,606 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting a88f0f4b45a547c7bcd742b8c51ec35d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148917002 2024-11-21T00:28:38,606 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:38,606 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a1b8f86ea34fdba424ceeee32777bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732148918366 2024-11-21T00:28:38,606 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:38,606 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:38,607 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:38,607 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/31a37b4b1a4b49b2a614b0d2673eac22, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/2fa66121d7bc4479b553e4519f586ce8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ec89fe03a0754b71a63f493b16cfffd5] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.7 K 2024-11-21T00:28:38,607 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31a37b4b1a4b49b2a614b0d2673eac22, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732148915286 2024-11-21T00:28:38,607 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa66121d7bc4479b553e4519f586ce8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148917002 2024-11-21T00:28:38,607 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec89fe03a0754b71a63f493b16cfffd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732148918366 2024-11-21T00:28:38,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/ac303524fcd0493080a51272094f73b3 is 50, key is test_row_0/A:col10/1732148918575/Put/seqid=0 2024-11-21T00:28:38,622 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:38,622 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/fd608ceaa5b7467d9b0aa299021daf07 is 50, key is test_row_0/A:col10/1732148918366/Put/seqid=0 2024-11-21T00:28:38,628 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:38,628 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/6da0c93b1c894c0c85fccf49de6b7554 is 50, key is test_row_0/B:col10/1732148918366/Put/seqid=0 2024-11-21T00:28:38,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742371_1547 (size=14741) 2024-11-21T00:28:38,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/ac303524fcd0493080a51272094f73b3 2024-11-21T00:28:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742372_1548 (size=13085) 2024-11-21T00:28:38,673 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/fd608ceaa5b7467d9b0aa299021daf07 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fd608ceaa5b7467d9b0aa299021daf07 2024-11-21T00:28:38,680 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into fd608ceaa5b7467d9b0aa299021daf07(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:38,680 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,680 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148918604; duration=0sec 2024-11-21T00:28:38,680 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:38,681 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:38,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742373_1549 (size=13085) 2024-11-21T00:28:38,681 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:28:38,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:28:38,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:28:38,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. because compaction request was cancelled 2024-11-21T00:28:38,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:38,691 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/6da0c93b1c894c0c85fccf49de6b7554 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/6da0c93b1c894c0c85fccf49de6b7554 2024-11-21T00:28:38,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/bb19f4079651438b97bcf8d67425dcc8 is 50, key is test_row_0/B:col10/1732148918575/Put/seqid=0 2024-11-21T00:28:38,704 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 6da0c93b1c894c0c85fccf49de6b7554(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:38,704 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:38,704 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148918604; duration=0sec 2024-11-21T00:28:38,704 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:38,704 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:38,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742374_1550 (size=12301) 2024-11-21T00:28:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148978807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:38,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148978922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T00:28:38,973 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:38,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T00:28:38,975 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:38,975 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:38,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:28:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:28:39,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:28:39,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:39,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:39,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:39,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/bb19f4079651438b97bcf8d67425dcc8 2024-11-21T00:28:39,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:39,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:39,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/2b0ae8e5308d480b8e94877451567f9e is 50, key is test_row_0/C:col10/1732148918575/Put/seqid=0 2024-11-21T00:28:39,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148979143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742375_1551 (size=12301) 2024-11-21T00:28:39,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/2b0ae8e5308d480b8e94877451567f9e 2024-11-21T00:28:39,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/ac303524fcd0493080a51272094f73b3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac303524fcd0493080a51272094f73b3 2024-11-21T00:28:39,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac303524fcd0493080a51272094f73b3, entries=200, sequenceid=340, filesize=14.4 K 2024-11-21T00:28:39,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/bb19f4079651438b97bcf8d67425dcc8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/bb19f4079651438b97bcf8d67425dcc8 2024-11-21T00:28:39,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/bb19f4079651438b97bcf8d67425dcc8, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T00:28:39,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/2b0ae8e5308d480b8e94877451567f9e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2b0ae8e5308d480b8e94877451567f9e 2024-11-21T00:28:39,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2b0ae8e5308d480b8e94877451567f9e, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T00:28:39,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5151f743f103348045956281ec3b4bb9 in 584ms, sequenceid=340, compaction requested=true 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:39,191 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:28:39,191 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:39,194 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. because compaction request was cancelled 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. because compaction request was cancelled 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:39,195 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:39,195 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:39,196 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/40f1fa75943749bf88651690867e7873, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d25491bbf8ac4fab9fc7fe871c558d20, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2b0ae8e5308d480b8e94877451567f9e] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.7 K 2024-11-21T00:28:39,196 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f1fa75943749bf88651690867e7873, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732148917002 2024-11-21T00:28:39,196 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d25491bbf8ac4fab9fc7fe871c558d20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732148918366 2024-11-21T00:28:39,197 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b0ae8e5308d480b8e94877451567f9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148918575 2024-11-21T00:28:39,206 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:39,206 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/2f9b182fa35046d4bba65715f89f883e is 50, key is test_row_0/C:col10/1732148918575/Put/seqid=0 2024-11-21T00:28:39,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742376_1552 (size=13119) 2024-11-21T00:28:39,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T00:28:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:39,285 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:39,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/ac0f79bd41b843d987ad34648d15058c is 50, key is test_row_0/A:col10/1732148918783/Put/seqid=0 2024-11-21T00:28:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:28:39,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742377_1553 (size=12301) 2024-11-21T00:28:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:39,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148979552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,566 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148979564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148979564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148979566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148979577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:28:39,645 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/2f9b182fa35046d4bba65715f89f883e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2f9b182fa35046d4bba65715f89f883e 2024-11-21T00:28:39,650 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into 2f9b182fa35046d4bba65715f89f883e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:39,650 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:39,650 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148919191; duration=0sec 2024-11-21T00:28:39,650 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:39,650 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148979682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148979682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,698 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/ac0f79bd41b843d987ad34648d15058c 2024-11-21T00:28:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148979692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148979692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/8b1bbbf471424ca0b2db2b570b85c9a3 is 50, key is test_row_0/B:col10/1732148918783/Put/seqid=0 2024-11-21T00:28:39,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742378_1554 (size=12301) 2024-11-21T00:28:39,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148979892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148979894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148979916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148979916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:28:40,143 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/8b1bbbf471424ca0b2db2b570b85c9a3 2024-11-21T00:28:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/5e91509bfecb4523a1dbc5cb5fd76c4d is 50, key is test_row_0/C:col10/1732148918783/Put/seqid=0 2024-11-21T00:28:40,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742379_1555 (size=12301) 2024-11-21T00:28:40,205 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/5e91509bfecb4523a1dbc5cb5fd76c4d 2024-11-21T00:28:40,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148980212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148980212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/ac0f79bd41b843d987ad34648d15058c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac0f79bd41b843d987ad34648d15058c 2024-11-21T00:28:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac0f79bd41b843d987ad34648d15058c, entries=150, sequenceid=361, filesize=12.0 K 2024-11-21T00:28:40,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/8b1bbbf471424ca0b2db2b570b85c9a3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/8b1bbbf471424ca0b2db2b570b85c9a3 2024-11-21T00:28:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148980233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,244 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/8b1bbbf471424ca0b2db2b570b85c9a3, entries=150, sequenceid=361, filesize=12.0 K 2024-11-21T00:28:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148980236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/5e91509bfecb4523a1dbc5cb5fd76c4d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/5e91509bfecb4523a1dbc5cb5fd76c4d 2024-11-21T00:28:40,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,261 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/5e91509bfecb4523a1dbc5cb5fd76c4d, entries=150, sequenceid=361, filesize=12.0 K 2024-11-21T00:28:40,262 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5151f743f103348045956281ec3b4bb9 in 978ms, sequenceid=361, compaction requested=true 2024-11-21T00:28:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T00:28:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T00:28:40,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T00:28:40,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-11-21T00:28:40,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2950 sec 2024-11-21T00:28:40,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:40,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:28:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:40,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/fa41ec871ce143d08b1ec5ea750d335a is 50, key is test_row_0/A:col10/1732148919564/Put/seqid=0 2024-11-21T00:28:40,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742380_1556 (size=14741) 2024-11-21T00:28:40,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/fa41ec871ce143d08b1ec5ea750d335a 2024-11-21T00:28:40,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/05cba84911004b1ba69efaa5b060b99f is 50, key is test_row_0/B:col10/1732148919564/Put/seqid=0 2024-11-21T00:28:40,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742381_1557 (size=12301) 2024-11-21T00:28:40,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148980789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148980796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148980808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148980811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148980912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148980924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148980925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148980925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T00:28:41,133 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:41,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T00:28:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:28:41,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:41,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:41,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:41,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148981133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148981152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148981152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148981152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/05cba84911004b1ba69efaa5b060b99f 2024-11-21T00:28:41,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/05f6e4ef1cea4d90af574f516b587a2a is 50, key is test_row_0/C:col10/1732148919564/Put/seqid=0 2024-11-21T00:28:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:28:41,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742382_1558 (size=12301) 2024-11-21T00:28:41,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/05f6e4ef1cea4d90af574f516b587a2a 2024-11-21T00:28:41,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/fa41ec871ce143d08b1ec5ea750d335a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fa41ec871ce143d08b1ec5ea750d335a 2024-11-21T00:28:41,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fa41ec871ce143d08b1ec5ea750d335a, entries=200, sequenceid=380, filesize=14.4 K 2024-11-21T00:28:41,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/05cba84911004b1ba69efaa5b060b99f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/05cba84911004b1ba69efaa5b060b99f 2024-11-21T00:28:41,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/05cba84911004b1ba69efaa5b060b99f, entries=150, sequenceid=380, filesize=12.0 K 2024-11-21T00:28:41,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/05f6e4ef1cea4d90af574f516b587a2a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/05f6e4ef1cea4d90af574f516b587a2a 2024-11-21T00:28:41,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/05f6e4ef1cea4d90af574f516b587a2a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-21T00:28:41,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5151f743f103348045956281ec3b4bb9 in 550ms, sequenceid=380, compaction requested=true 2024-11-21T00:28:41,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:41,288 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:41,289 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:28:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:41,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T00:28:41,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:41,293 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T00:28:41,294 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:41,294 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:41,294 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:41,294 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fd608ceaa5b7467d9b0aa299021daf07, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac303524fcd0493080a51272094f73b3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac0f79bd41b843d987ad34648d15058c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fa41ec871ce143d08b1ec5ea750d335a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=53.6 K 2024-11-21T00:28:41,295 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:28:41,295 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:41,295 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:41,295 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/6da0c93b1c894c0c85fccf49de6b7554, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/bb19f4079651438b97bcf8d67425dcc8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/8b1bbbf471424ca0b2db2b570b85c9a3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/05cba84911004b1ba69efaa5b060b99f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=48.8 K 2024-11-21T00:28:41,295 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da0c93b1c894c0c85fccf49de6b7554, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732148918366 2024-11-21T00:28:41,295 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd608ceaa5b7467d9b0aa299021daf07, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732148918366 2024-11-21T00:28:41,296 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bb19f4079651438b97bcf8d67425dcc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148918575 2024-11-21T00:28:41,296 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac303524fcd0493080a51272094f73b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148918575 2024-11-21T00:28:41,297 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1bbbf471424ca0b2db2b570b85c9a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732148918772 2024-11-21T00:28:41,297 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0f79bd41b843d987ad34648d15058c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732148918772 2024-11-21T00:28:41,299 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa41ec871ce143d08b1ec5ea750d335a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732148919539 2024-11-21T00:28:41,300 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 05cba84911004b1ba69efaa5b060b99f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732148919564 2024-11-21T00:28:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/86cdcaaa00d24224a5f6375b60103618 is 50, key is test_row_0/A:col10/1732148920791/Put/seqid=0 2024-11-21T00:28:41,344 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:41,344 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/b0ab4f804b42497094741ac76c589674 is 50, key is test_row_0/B:col10/1732148919564/Put/seqid=0 2024-11-21T00:28:41,346 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:41,347 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/66305bd7e4854f998697cf5cbadcc0b3 is 50, key is test_row_0/A:col10/1732148919564/Put/seqid=0 2024-11-21T00:28:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742383_1559 (size=12301) 2024-11-21T00:28:41,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742384_1560 (size=13221) 2024-11-21T00:28:41,383 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/66305bd7e4854f998697cf5cbadcc0b3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/66305bd7e4854f998697cf5cbadcc0b3 2024-11-21T00:28:41,391 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into 66305bd7e4854f998697cf5cbadcc0b3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:41,391 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:41,391 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=12, startTime=1732148921288; duration=0sec 2024-11-21T00:28:41,391 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:41,391 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:41,391 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:41,392 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:41,392 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:41,392 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:41,392 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2f9b182fa35046d4bba65715f89f883e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/5e91509bfecb4523a1dbc5cb5fd76c4d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/05f6e4ef1cea4d90af574f516b587a2a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.8 K 2024-11-21T00:28:41,393 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f9b182fa35046d4bba65715f89f883e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732148918575 2024-11-21T00:28:41,398 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e91509bfecb4523a1dbc5cb5fd76c4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732148918772 2024-11-21T00:28:41,398 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f6e4ef1cea4d90af574f516b587a2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732148919564 2024-11-21T00:28:41,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742385_1561 (size=13221) 2024-11-21T00:28:41,430 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:41,431 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/b0ab4f804b42497094741ac76c589674 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b0ab4f804b42497094741ac76c589674 2024-11-21T00:28:41,431 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/b7e3ce69db0442178713c3a1beaf6b69 is 50, key is test_row_0/C:col10/1732148919564/Put/seqid=0 2024-11-21T00:28:41,435 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into b0ab4f804b42497094741ac76c589674(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:41,435 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:41,435 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=12, startTime=1732148921289; duration=0sec 2024-11-21T00:28:41,435 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:41,435 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:41,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742386_1562 (size=13221) 2024-11-21T00:28:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:28:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:41,459 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/b7e3ce69db0442178713c3a1beaf6b69 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b7e3ce69db0442178713c3a1beaf6b69 2024-11-21T00:28:41,473 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into b7e3ce69db0442178713c3a1beaf6b69(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:41,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:41,473 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148921289; duration=0sec 2024-11-21T00:28:41,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:41,473 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:41,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148981507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148981508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148981509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148981510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148981622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148981633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148981636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148981633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:28:41,779 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/86cdcaaa00d24224a5f6375b60103618 2024-11-21T00:28:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/f296d5b14a874c1f8699b779442b5f77 is 50, key is test_row_0/B:col10/1732148920791/Put/seqid=0 2024-11-21T00:28:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742387_1563 (size=12301) 2024-11-21T00:28:41,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148981842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148981842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148981856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:41,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148981862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148982152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148982152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148982172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148982182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,213 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/f296d5b14a874c1f8699b779442b5f77 2024-11-21T00:28:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1a867553f2ab42449afe02524c1854f8 is 50, key is test_row_0/C:col10/1732148920791/Put/seqid=0 2024-11-21T00:28:42,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742388_1564 (size=12301) 2024-11-21T00:28:42,256 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1a867553f2ab42449afe02524c1854f8 2024-11-21T00:28:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:28:42,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/86cdcaaa00d24224a5f6375b60103618 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/86cdcaaa00d24224a5f6375b60103618 2024-11-21T00:28:42,296 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/86cdcaaa00d24224a5f6375b60103618, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T00:28:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/f296d5b14a874c1f8699b779442b5f77 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f296d5b14a874c1f8699b779442b5f77 2024-11-21T00:28:42,304 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f296d5b14a874c1f8699b779442b5f77, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T00:28:42,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/1a867553f2ab42449afe02524c1854f8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1a867553f2ab42449afe02524c1854f8 2024-11-21T00:28:42,312 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1a867553f2ab42449afe02524c1854f8, entries=150, sequenceid=398, filesize=12.0 K 2024-11-21T00:28:42,313 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5151f743f103348045956281ec3b4bb9 in 1020ms, sequenceid=398, compaction requested=false 2024-11-21T00:28:42,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:42,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:42,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T00:28:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T00:28:42,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T00:28:42,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-21T00:28:42,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1910 sec 2024-11-21T00:28:42,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:42,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:42,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/7ca6a92286ed4c2ab6f9e3c846b96d72 is 50, key is test_row_0/A:col10/1732148922682/Put/seqid=0 2024-11-21T00:28:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742389_1565 (size=14741) 2024-11-21T00:28:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148982751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148982754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148982771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148982883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148982883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148982883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148982894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148983105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148983105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148983105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148983112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/7ca6a92286ed4c2ab6f9e3c846b96d72 2024-11-21T00:28:43,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c45f43fe2f784736b148405d16c7689b is 50, key is test_row_0/B:col10/1732148922682/Put/seqid=0 2024-11-21T00:28:43,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742390_1566 (size=12301) 2024-11-21T00:28:43,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c45f43fe2f784736b148405d16c7689b 2024-11-21T00:28:43,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/659df8da369a4bef91090df1b7d2e4aa is 50, key is test_row_0/C:col10/1732148922682/Put/seqid=0 2024-11-21T00:28:43,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742391_1567 (size=12301) 2024-11-21T00:28:43,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/659df8da369a4bef91090df1b7d2e4aa 2024-11-21T00:28:43,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/7ca6a92286ed4c2ab6f9e3c846b96d72 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7ca6a92286ed4c2ab6f9e3c846b96d72 2024-11-21T00:28:43,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7ca6a92286ed4c2ab6f9e3c846b96d72, entries=200, sequenceid=420, filesize=14.4 K 2024-11-21T00:28:43,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/c45f43fe2f784736b148405d16c7689b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c45f43fe2f784736b148405d16c7689b 2024-11-21T00:28:43,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c45f43fe2f784736b148405d16c7689b, entries=150, sequenceid=420, filesize=12.0 K 2024-11-21T00:28:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T00:28:43,282 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:43,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:43,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/659df8da369a4bef91090df1b7d2e4aa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/659df8da369a4bef91090df1b7d2e4aa 2024-11-21T00:28:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T00:28:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:28:43,290 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:43,290 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:43,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:43,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/659df8da369a4bef91090df1b7d2e4aa, entries=150, sequenceid=420, filesize=12.0 K 2024-11-21T00:28:43,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5151f743f103348045956281ec3b4bb9 in 621ms, sequenceid=420, compaction requested=true 2024-11-21T00:28:43,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:43,305 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:28:43,305 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:43,308 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:43,309 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:43,309 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:43,309 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b0ab4f804b42497094741ac76c589674, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f296d5b14a874c1f8699b779442b5f77, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c45f43fe2f784736b148405d16c7689b] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.9 K 2024-11-21T00:28:43,310 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ab4f804b42497094741ac76c589674, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732148919564 2024-11-21T00:28:43,312 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:43,312 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f296d5b14a874c1f8699b779442b5f77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732148920786 2024-11-21T00:28:43,312 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:43,313 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:43,313 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/66305bd7e4854f998697cf5cbadcc0b3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/86cdcaaa00d24224a5f6375b60103618, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7ca6a92286ed4c2ab6f9e3c846b96d72] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=39.3 K 2024-11-21T00:28:43,313 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45f43fe2f784736b148405d16c7689b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732148921508 2024-11-21T00:28:43,313 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 66305bd7e4854f998697cf5cbadcc0b3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732148919564 2024-11-21T00:28:43,313 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 86cdcaaa00d24224a5f6375b60103618, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732148920786 2024-11-21T00:28:43,314 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca6a92286ed4c2ab6f9e3c846b96d72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732148921500 2024-11-21T00:28:43,338 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:43,338 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/f7e9cc2b47274ef48fd194ac81634d50 is 50, key is test_row_0/A:col10/1732148922682/Put/seqid=0 2024-11-21T00:28:43,354 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:43,355 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/3b8e7ad880664edb9bedef8f5b660f25 is 50, key is test_row_0/B:col10/1732148922682/Put/seqid=0 2024-11-21T00:28:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:28:43,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742393_1569 (size=13323) 2024-11-21T00:28:43,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742392_1568 (size=13323) 2024-11-21T00:28:43,433 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/3b8e7ad880664edb9bedef8f5b660f25 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/3b8e7ad880664edb9bedef8f5b660f25 2024-11-21T00:28:43,439 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 3b8e7ad880664edb9bedef8f5b660f25(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:43,439 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:43,439 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148923305; duration=0sec 2024-11-21T00:28:43,439 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:43,439 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:43,439 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:43,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:43,440 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:43,440 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:43,440 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b7e3ce69db0442178713c3a1beaf6b69, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1a867553f2ab42449afe02524c1854f8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/659df8da369a4bef91090df1b7d2e4aa] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=36.9 K 2024-11-21T00:28:43,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7e3ce69db0442178713c3a1beaf6b69, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732148919564 2024-11-21T00:28:43,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a867553f2ab42449afe02524c1854f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732148920786 2024-11-21T00:28:43,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 659df8da369a4bef91090df1b7d2e4aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732148921508 2024-11-21T00:28:43,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T00:28:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:43,448 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:28:43,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:43,454 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:43,455 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/c0fe38492df048b78760e2fa6bce251f is 50, key is test_row_0/C:col10/1732148922682/Put/seqid=0 2024-11-21T00:28:43,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/51fb03a68eba4480bd5f87178ec63313 is 50, key is test_row_0/A:col10/1732148923447/Put/seqid=0 2024-11-21T00:28:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742394_1570 (size=13323) 2024-11-21T00:28:43,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742395_1571 (size=12301) 2024-11-21T00:28:43,501 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/51fb03a68eba4480bd5f87178ec63313 2024-11-21T00:28:43,503 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/c0fe38492df048b78760e2fa6bce251f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/c0fe38492df048b78760e2fa6bce251f 2024-11-21T00:28:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148983496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,518 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into c0fe38492df048b78760e2fa6bce251f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:43,518 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:43,518 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148923305; duration=0sec 2024-11-21T00:28:43,518 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:43,518 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:43,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9abacd09336442168650d2b1529f6cf9 is 50, key is test_row_0/B:col10/1732148923447/Put/seqid=0 2024-11-21T00:28:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742396_1572 (size=12301) 2024-11-21T00:28:43,524 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9abacd09336442168650d2b1529f6cf9 2024-11-21T00:28:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ec42155fdcee4dffab5d9f2601a256ca is 50, key is test_row_0/C:col10/1732148923447/Put/seqid=0 2024-11-21T00:28:43,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148983516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148983516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148983518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742397_1573 (size=12301) 2024-11-21T00:28:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44666 deadline: 1732148983583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,595 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 5151f743f103348045956281ec3b4bb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:28:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:28:43,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148983622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148983643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148983643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148983643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,840 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/f7e9cc2b47274ef48fd194ac81634d50 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f7e9cc2b47274ef48fd194ac81634d50 2024-11-21T00:28:43,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148983842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,851 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into f7e9cc2b47274ef48fd194ac81634d50(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:43,852 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:43,852 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148923304; duration=0sec 2024-11-21T00:28:43,852 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:43,852 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148983862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148983862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148983864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:28:43,969 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ec42155fdcee4dffab5d9f2601a256ca 2024-11-21T00:28:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/51fb03a68eba4480bd5f87178ec63313 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/51fb03a68eba4480bd5f87178ec63313 2024-11-21T00:28:43,979 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/51fb03a68eba4480bd5f87178ec63313, entries=150, sequenceid=439, filesize=12.0 K 2024-11-21T00:28:43,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/9abacd09336442168650d2b1529f6cf9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9abacd09336442168650d2b1529f6cf9 2024-11-21T00:28:43,984 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9abacd09336442168650d2b1529f6cf9, entries=150, sequenceid=439, filesize=12.0 K 2024-11-21T00:28:43,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ec42155fdcee4dffab5d9f2601a256ca as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ec42155fdcee4dffab5d9f2601a256ca 2024-11-21T00:28:43,988 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ec42155fdcee4dffab5d9f2601a256ca, entries=150, sequenceid=439, filesize=12.0 K 2024-11-21T00:28:43,989 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5151f743f103348045956281ec3b4bb9 in 541ms, sequenceid=439, compaction requested=false 2024-11-21T00:28:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T00:28:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T00:28:43,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T00:28:43,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 701 msec 2024-11-21T00:28:43,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 711 msec 2024-11-21T00:28:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:44,157 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:44,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T00:28:44,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/3ce7c717941c4bb289c10bc279c161b4 is 50, key is test_row_0/A:col10/1732148923491/Put/seqid=0 2024-11-21T00:28:44,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742398_1574 (size=14741) 2024-11-21T00:28:44,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148984209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148984209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148984216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148984218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148984322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148984331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148984332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148984332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T00:28:44,430 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:44,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T00:28:44,437 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:28:44,438 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:44,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:28:44,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148984544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148984552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148984552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148984552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/3ce7c717941c4bb289c10bc279c161b4 2024-11-21T00:28:44,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/4016b613ec1643ce9ab970f58b4c8958 is 50, key is test_row_0/B:col10/1732148923491/Put/seqid=0 2024-11-21T00:28:44,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742399_1575 (size=12301) 2024-11-21T00:28:44,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:44,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:44,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:28:44,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148984862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148984872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148984873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148984873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:44,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:44,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:44,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/4016b613ec1643ce9ab970f58b4c8958 2024-11-21T00:28:45,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/df99d88fac514ca98ac6c16feff2f925 is 50, key is test_row_0/C:col10/1732148923491/Put/seqid=0 2024-11-21T00:28:45,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742400_1576 (size=12301) 2024-11-21T00:28:45,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:45,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:28:45,081 INFO [Thread-2139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2139. 2024-11-21T00:28:45,081 DEBUG [Thread-2139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:45,081 DEBUG [Thread-2139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,081 DEBUG [Thread-2139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:45,089 INFO [Thread-2143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2143. 2024-11-21T00:28:45,089 DEBUG [Thread-2143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:45,089 DEBUG [Thread-2143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:45,091 DEBUG [Thread-2143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,091 INFO [Thread-2145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2145. 2024-11-21T00:28:45,091 DEBUG [Thread-2145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:45,091 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,091 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:45,092 INFO [Thread-2141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2141. 2024-11-21T00:28:45,092 DEBUG [Thread-2141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:45,092 DEBUG [Thread-2141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,092 DEBUG [Thread-2141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:45,103 INFO [Thread-2147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2147. 2024-11-21T00:28:45,103 DEBUG [Thread-2147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:45,103 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,103 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:45,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:45,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. as already flushing 2024-11-21T00:28:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44738 deadline: 1732148985382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44716 deadline: 1732148985382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44706 deadline: 1732148985382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:45,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44704 deadline: 1732148985392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 , the old value is region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=2 because the exception is null or not the one we care about 2024-11-21T00:28:45,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/df99d88fac514ca98ac6c16feff2f925 2024-11-21T00:28:45,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/3ce7c717941c4bb289c10bc279c161b4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3ce7c717941c4bb289c10bc279c161b4 2024-11-21T00:28:45,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3ce7c717941c4bb289c10bc279c161b4, entries=200, sequenceid=460, filesize=14.4 K 2024-11-21T00:28:45,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/4016b613ec1643ce9ab970f58b4c8958 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4016b613ec1643ce9ab970f58b4c8958 2024-11-21T00:28:45,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4016b613ec1643ce9ab970f58b4c8958, entries=150, sequenceid=460, filesize=12.0 K 2024-11-21T00:28:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/df99d88fac514ca98ac6c16feff2f925 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/df99d88fac514ca98ac6c16feff2f925 2024-11-21T00:28:45,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/df99d88fac514ca98ac6c16feff2f925, entries=150, sequenceid=460, filesize=12.0 K 2024-11-21T00:28:45,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5151f743f103348045956281ec3b4bb9 in 1276ms, sequenceid=460, compaction requested=true 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:45,433 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:45,433 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5151f743f103348045956281ec3b4bb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:28:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:45,434 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:45,434 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:45,434 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/A is initiating minor compaction (all files) 2024-11-21T00:28:45,434 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/B is initiating minor compaction (all files) 2024-11-21T00:28:45,434 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/A in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,434 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/B in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,434 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f7e9cc2b47274ef48fd194ac81634d50, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/51fb03a68eba4480bd5f87178ec63313, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3ce7c717941c4bb289c10bc279c161b4] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=39.4 K 2024-11-21T00:28:45,434 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/3b8e7ad880664edb9bedef8f5b660f25, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9abacd09336442168650d2b1529f6cf9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4016b613ec1643ce9ab970f58b4c8958] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=37.0 K 2024-11-21T00:28:45,434 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b8e7ad880664edb9bedef8f5b660f25, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732148921508 2024-11-21T00:28:45,434 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7e9cc2b47274ef48fd194ac81634d50, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732148921508 2024-11-21T00:28:45,435 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51fb03a68eba4480bd5f87178ec63313, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732148922751 2024-11-21T00:28:45,435 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abacd09336442168650d2b1529f6cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732148922751 2024-11-21T00:28:45,435 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce7c717941c4bb289c10bc279c161b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732148923491 2024-11-21T00:28:45,435 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4016b613ec1643ce9ab970f58b4c8958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732148923491 2024-11-21T00:28:45,441 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#A#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:45,441 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:45,441 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/1c1cddccd63b492cb216fcb6b8963a42 is 50, key is test_row_0/B:col10/1732148923491/Put/seqid=0 2024-11-21T00:28:45,441 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/1bd5a451346047558f781f6607914dd8 is 50, key is test_row_0/A:col10/1732148923491/Put/seqid=0 2024-11-21T00:28:45,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742401_1577 (size=13425) 2024-11-21T00:28:45,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742402_1578 (size=13425) 2024-11-21T00:28:45,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T00:28:45,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,516 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T00:28:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/5e0b0d4334714940b5c574a98570eab3 is 50, key is test_row_0/A:col10/1732148924204/Put/seqid=0 2024-11-21T00:28:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742403_1579 (size=12301) 2024-11-21T00:28:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:28:45,857 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/1bd5a451346047558f781f6607914dd8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1bd5a451346047558f781f6607914dd8 2024-11-21T00:28:45,857 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/1c1cddccd63b492cb216fcb6b8963a42 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/1c1cddccd63b492cb216fcb6b8963a42 2024-11-21T00:28:45,863 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/A of 5151f743f103348045956281ec3b4bb9 into 1bd5a451346047558f781f6607914dd8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:45,863 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:45,863 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/A, priority=13, startTime=1732148925433; duration=0sec 2024-11-21T00:28:45,863 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:28:45,863 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:A 2024-11-21T00:28:45,863 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:28:45,864 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/B of 5151f743f103348045956281ec3b4bb9 into 1c1cddccd63b492cb216fcb6b8963a42(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:45,864 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:45,864 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/B, priority=13, startTime=1732148925433; duration=0sec 2024-11-21T00:28:45,864 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:45,864 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:B 2024-11-21T00:28:45,872 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:28:45,873 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 5151f743f103348045956281ec3b4bb9/C is initiating minor compaction (all files) 2024-11-21T00:28:45,873 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5151f743f103348045956281ec3b4bb9/C in TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:45,873 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/c0fe38492df048b78760e2fa6bce251f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ec42155fdcee4dffab5d9f2601a256ca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/df99d88fac514ca98ac6c16feff2f925] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp, totalSize=37.0 K 2024-11-21T00:28:45,876 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0fe38492df048b78760e2fa6bce251f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732148921508 2024-11-21T00:28:45,877 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec42155fdcee4dffab5d9f2601a256ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732148922751 2024-11-21T00:28:45,882 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting df99d88fac514ca98ac6c16feff2f925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732148923491 2024-11-21T00:28:45,896 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5151f743f103348045956281ec3b4bb9#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:28:45,897 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/104f70a6516f4716991291c55bcfdfde is 50, key is test_row_0/C:col10/1732148923491/Put/seqid=0 2024-11-21T00:28:45,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742404_1580 (size=13425) 2024-11-21T00:28:45,915 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/104f70a6516f4716991291c55bcfdfde as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/104f70a6516f4716991291c55bcfdfde 2024-11-21T00:28:45,923 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5151f743f103348045956281ec3b4bb9/C of 5151f743f103348045956281ec3b4bb9 into 104f70a6516f4716991291c55bcfdfde(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:28:45,923 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:45,923 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., storeName=5151f743f103348045956281ec3b4bb9/C, priority=13, startTime=1732148925433; duration=0sec 2024-11-21T00:28:45,923 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:28:45,923 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5151f743f103348045956281ec3b4bb9:C 2024-11-21T00:28:45,926 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/5e0b0d4334714940b5c574a98570eab3 2024-11-21T00:28:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/86994df8f5e24991ab7507a65be8c88c is 50, key is test_row_0/B:col10/1732148924204/Put/seqid=0 2024-11-21T00:28:45,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742405_1581 (size=12301) 2024-11-21T00:28:46,007 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:28:46,350 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/86994df8f5e24991ab7507a65be8c88c 2024-11-21T00:28:46,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ff6cb3d2e0474a68ae618697f633f1aa is 50, key is test_row_0/C:col10/1732148924204/Put/seqid=0 2024-11-21T00:28:46,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742406_1582 (size=12301) 2024-11-21T00:28:46,368 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ff6cb3d2e0474a68ae618697f633f1aa 2024-11-21T00:28:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/5e0b0d4334714940b5c574a98570eab3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/5e0b0d4334714940b5c574a98570eab3 2024-11-21T00:28:46,375 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/5e0b0d4334714940b5c574a98570eab3, entries=150, sequenceid=477, filesize=12.0 K 2024-11-21T00:28:46,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/86994df8f5e24991ab7507a65be8c88c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/86994df8f5e24991ab7507a65be8c88c 2024-11-21T00:28:46,382 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/86994df8f5e24991ab7507a65be8c88c, entries=150, sequenceid=477, filesize=12.0 K 2024-11-21T00:28:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/ff6cb3d2e0474a68ae618697f633f1aa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ff6cb3d2e0474a68ae618697f633f1aa 2024-11-21T00:28:46,390 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ff6cb3d2e0474a68ae618697f633f1aa, entries=150, sequenceid=477, filesize=12.0 K 2024-11-21T00:28:46,391 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 5151f743f103348045956281ec3b4bb9 in 875ms, sequenceid=477, compaction requested=false 2024-11-21T00:28:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5151f743f103348045956281ec3b4bb9: 2024-11-21T00:28:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T00:28:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T00:28:46,394 INFO [Thread-2130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2130. 2024-11-21T00:28:46,394 INFO [Thread-2128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2128. 2024-11-21T00:28:46,394 DEBUG [Thread-2130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:46,394 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,394 DEBUG [Thread-2128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:46,394 DEBUG [Thread-2128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,394 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:46,394 DEBUG [Thread-2128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:46,398 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-21T00:28:46,398 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:46,398 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:46,399 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T00:28:46,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9570 sec 2024-11-21T00:28:46,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.9670 sec 2024-11-21T00:28:46,404 INFO [Thread-2132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2132. 2024-11-21T00:28:46,404 DEBUG [Thread-2132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:46,404 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,404 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:46,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T00:28:46,582 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:53,653 INFO [Thread-2134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2134. 2024-11-21T00:28:53,653 DEBUG [Thread-2134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:28:53,653 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:53,653 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:53,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 899 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2697 rows 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 897 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2691 rows 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 897 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2691 rows 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 904 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2694 rows 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 905 2024-11-21T00:28:53,654 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2714 rows 2024-11-21T00:28:53,654 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:28:53,654 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:28:53,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:53,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:53,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:28:53,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:28:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:28:53,659 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148933659"}]},"ts":"1732148933659"} 2024-11-21T00:28:53,660 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:28:53,660 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:28:53,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:28:53,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, UNASSIGN}] 2024-11-21T00:28:53,665 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, UNASSIGN 2024-11-21T00:28:53,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=5151f743f103348045956281ec3b4bb9, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:53,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, UNASSIGN because future has completed 2024-11-21T00:28:53,667 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:28:53,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:28:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:28:53,819 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:53,820 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:28:53,820 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 5151f743f103348045956281ec3b4bb9, disabling compactions & flushes 2024-11-21T00:28:53,820 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:53,820 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:53,820 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. after waiting 0 ms 2024-11-21T00:28:53,820 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:53,820 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 5151f743f103348045956281ec3b4bb9 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T00:28:53,826 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/f22ed81ca6754f279ab83234ad6a9062 is 50, key is test_row_1/A:col10/1732148933652/Put/seqid=0 2024-11-21T00:28:53,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742407_1583 (size=9857) 2024-11-21T00:28:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:28:54,237 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/f22ed81ca6754f279ab83234ad6a9062 2024-11-21T00:28:54,269 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/488c4aebb9794f2c8f254111e8bec1ea is 50, key is test_row_1/B:col10/1732148933652/Put/seqid=0 2024-11-21T00:28:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:28:54,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742408_1584 (size=9857) 2024-11-21T00:28:54,688 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/488c4aebb9794f2c8f254111e8bec1ea 2024-11-21T00:28:54,698 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/75a1822890fa415f8cab655be4dfcdf7 is 50, key is test_row_1/C:col10/1732148933652/Put/seqid=0 2024-11-21T00:28:54,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742409_1585 (size=9857) 2024-11-21T00:28:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:28:55,134 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/75a1822890fa415f8cab655be4dfcdf7 2024-11-21T00:28:55,146 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/A/f22ed81ca6754f279ab83234ad6a9062 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f22ed81ca6754f279ab83234ad6a9062 2024-11-21T00:28:55,153 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f22ed81ca6754f279ab83234ad6a9062, entries=100, sequenceid=488, filesize=9.6 K 2024-11-21T00:28:55,154 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/B/488c4aebb9794f2c8f254111e8bec1ea as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/488c4aebb9794f2c8f254111e8bec1ea 2024-11-21T00:28:55,158 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/488c4aebb9794f2c8f254111e8bec1ea, entries=100, sequenceid=488, filesize=9.6 K 2024-11-21T00:28:55,159 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/.tmp/C/75a1822890fa415f8cab655be4dfcdf7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/75a1822890fa415f8cab655be4dfcdf7 2024-11-21T00:28:55,164 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/75a1822890fa415f8cab655be4dfcdf7, entries=100, sequenceid=488, filesize=9.6 K 2024-11-21T00:28:55,169 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5151f743f103348045956281ec3b4bb9 in 1349ms, sequenceid=488, compaction requested=true 2024-11-21T00:28:55,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7efe8a6374124cdd9d7e1c384935dedc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/8c83cfc17ed541b1bc220ad00412b8a2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/be1f0bd3246e44eba9db2b82d096f1d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/bfa39c3ff42e458b86d1551b03b76313, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a58983ae938144e1810007e1cbdb7578, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/68cc4db6c1434d6caafcbe675e1cab14, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/379a1013b7bb42ada082fb2c04c14ac6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/633c9ed9310c4bf4be4c60f6afd6d9ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e635976b175a4ea29225e1f18d707040, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/0fdbe3c1c6934353bf9e1004c7186077, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3181e0b310a84e1eb70149b09a678621, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1a4c48964ce7488bb2988440bdaf1f6b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/b7586015d070436cad233a6466385e2d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/71764c05aea14d9cb8c13fbe670fd8bf, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/438b2b50eeec440b86d042442bf6c29c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6c0c4fcc4bc94b4f9a4de846325b2400, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6864aa62af8a4f548f4df97ef3fd8cc0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/29b24d12eaca400fbf49be7980b76062, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/61b61c1dd9f749c6979c034a92783de1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e79d7d1a4c524160becbd2e37b4a3e1b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a88f0f4b45a547c7bcd742b8c51ec35d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fd608ceaa5b7467d9b0aa299021daf07, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e3a1b8f86ea34fdba424ceeee32777bc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac303524fcd0493080a51272094f73b3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac0f79bd41b843d987ad34648d15058c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fa41ec871ce143d08b1ec5ea750d335a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/66305bd7e4854f998697cf5cbadcc0b3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/86cdcaaa00d24224a5f6375b60103618, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7ca6a92286ed4c2ab6f9e3c846b96d72, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f7e9cc2b47274ef48fd194ac81634d50, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/51fb03a68eba4480bd5f87178ec63313, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3ce7c717941c4bb289c10bc279c161b4] to archive 2024-11-21T00:28:55,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:28:55,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7efe8a6374124cdd9d7e1c384935dedc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7efe8a6374124cdd9d7e1c384935dedc 2024-11-21T00:28:55,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/8c83cfc17ed541b1bc220ad00412b8a2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/8c83cfc17ed541b1bc220ad00412b8a2 2024-11-21T00:28:55,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/be1f0bd3246e44eba9db2b82d096f1d1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/be1f0bd3246e44eba9db2b82d096f1d1 2024-11-21T00:28:55,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/bfa39c3ff42e458b86d1551b03b76313 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/bfa39c3ff42e458b86d1551b03b76313 2024-11-21T00:28:55,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a58983ae938144e1810007e1cbdb7578 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a58983ae938144e1810007e1cbdb7578 2024-11-21T00:28:55,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/68cc4db6c1434d6caafcbe675e1cab14 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/68cc4db6c1434d6caafcbe675e1cab14 2024-11-21T00:28:55,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/379a1013b7bb42ada082fb2c04c14ac6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/379a1013b7bb42ada082fb2c04c14ac6 2024-11-21T00:28:55,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a9cfc6d0261a4c0fb1b0ce69384ae0f9 2024-11-21T00:28:55,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/633c9ed9310c4bf4be4c60f6afd6d9ed to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/633c9ed9310c4bf4be4c60f6afd6d9ed 2024-11-21T00:28:55,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e635976b175a4ea29225e1f18d707040 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e635976b175a4ea29225e1f18d707040 2024-11-21T00:28:55,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/0fdbe3c1c6934353bf9e1004c7186077 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/0fdbe3c1c6934353bf9e1004c7186077 2024-11-21T00:28:55,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3181e0b310a84e1eb70149b09a678621 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3181e0b310a84e1eb70149b09a678621 2024-11-21T00:28:55,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1a4c48964ce7488bb2988440bdaf1f6b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1a4c48964ce7488bb2988440bdaf1f6b 2024-11-21T00:28:55,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/b7586015d070436cad233a6466385e2d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/b7586015d070436cad233a6466385e2d 2024-11-21T00:28:55,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/71764c05aea14d9cb8c13fbe670fd8bf to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/71764c05aea14d9cb8c13fbe670fd8bf 2024-11-21T00:28:55,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/438b2b50eeec440b86d042442bf6c29c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/438b2b50eeec440b86d042442bf6c29c 2024-11-21T00:28:55,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6c0c4fcc4bc94b4f9a4de846325b2400 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6c0c4fcc4bc94b4f9a4de846325b2400 2024-11-21T00:28:55,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6864aa62af8a4f548f4df97ef3fd8cc0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/6864aa62af8a4f548f4df97ef3fd8cc0 2024-11-21T00:28:55,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/29b24d12eaca400fbf49be7980b76062 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/29b24d12eaca400fbf49be7980b76062 2024-11-21T00:28:55,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/61b61c1dd9f749c6979c034a92783de1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/61b61c1dd9f749c6979c034a92783de1 2024-11-21T00:28:55,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e79d7d1a4c524160becbd2e37b4a3e1b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e79d7d1a4c524160becbd2e37b4a3e1b 2024-11-21T00:28:55,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a88f0f4b45a547c7bcd742b8c51ec35d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/a88f0f4b45a547c7bcd742b8c51ec35d 2024-11-21T00:28:55,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fd608ceaa5b7467d9b0aa299021daf07 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fd608ceaa5b7467d9b0aa299021daf07 2024-11-21T00:28:55,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e3a1b8f86ea34fdba424ceeee32777bc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/e3a1b8f86ea34fdba424ceeee32777bc 2024-11-21T00:28:55,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac303524fcd0493080a51272094f73b3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac303524fcd0493080a51272094f73b3 2024-11-21T00:28:55,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac0f79bd41b843d987ad34648d15058c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/ac0f79bd41b843d987ad34648d15058c 2024-11-21T00:28:55,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fa41ec871ce143d08b1ec5ea750d335a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/fa41ec871ce143d08b1ec5ea750d335a 2024-11-21T00:28:55,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/66305bd7e4854f998697cf5cbadcc0b3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/66305bd7e4854f998697cf5cbadcc0b3 2024-11-21T00:28:55,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/86cdcaaa00d24224a5f6375b60103618 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/86cdcaaa00d24224a5f6375b60103618 2024-11-21T00:28:55,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7ca6a92286ed4c2ab6f9e3c846b96d72 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/7ca6a92286ed4c2ab6f9e3c846b96d72 2024-11-21T00:28:55,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f7e9cc2b47274ef48fd194ac81634d50 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f7e9cc2b47274ef48fd194ac81634d50 2024-11-21T00:28:55,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/51fb03a68eba4480bd5f87178ec63313 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/51fb03a68eba4480bd5f87178ec63313 2024-11-21T00:28:55,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3ce7c717941c4bb289c10bc279c161b4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/3ce7c717941c4bb289c10bc279c161b4 2024-11-21T00:28:55,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4d1720e1e61c4a16b0fcbaa701cfbe5a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/eaaca4ff49674e53b77c4be5007f7610, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/29039000a9254a74bc7ce2d3b6408867, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c97b2e23055b43c6b6e749281a4094f0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/7a7f430638b84f409f13d48f1b384d04, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9139fa6d44714023a87cbd179127b0ce, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c1fab06f307f4d7392310f509d47a9ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/afd57d1e5182420fa06dc3799ed54700, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b9ebbb6982244c569c1aebd0d257cc9c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/659841b7c4674892bec338160b4258b4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9ba3570c90674e4f91034420d5f0e18a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/a67761ac84a54727bb78c51f96709b64, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/014919fc5a2b439e98b4e2b85222f061, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/273c2afe2e4341089a2ac7eab77cafd8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ef4f086726b1455d870def7bc7f98167, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/10d479665f5542a2b4a8e97ee9970e30, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/33ca0b79d3f4431e98950601f6b7d4da, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f207c279c89049f68bab8fb191dd149b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/21109253651d492f89cf86f5a73c085c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/31a37b4b1a4b49b2a614b0d2673eac22, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/12ff07f290fe4443a966f4a97e05e23a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/2fa66121d7bc4479b553e4519f586ce8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/6da0c93b1c894c0c85fccf49de6b7554, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ec89fe03a0754b71a63f493b16cfffd5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/bb19f4079651438b97bcf8d67425dcc8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/8b1bbbf471424ca0b2db2b570b85c9a3, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b0ab4f804b42497094741ac76c589674, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/05cba84911004b1ba69efaa5b060b99f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f296d5b14a874c1f8699b779442b5f77, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/3b8e7ad880664edb9bedef8f5b660f25, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c45f43fe2f784736b148405d16c7689b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9abacd09336442168650d2b1529f6cf9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4016b613ec1643ce9ab970f58b4c8958] to archive 2024-11-21T00:28:55,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:28:55,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4d1720e1e61c4a16b0fcbaa701cfbe5a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4d1720e1e61c4a16b0fcbaa701cfbe5a 2024-11-21T00:28:55,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/eaaca4ff49674e53b77c4be5007f7610 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/eaaca4ff49674e53b77c4be5007f7610 2024-11-21T00:28:55,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/29039000a9254a74bc7ce2d3b6408867 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/29039000a9254a74bc7ce2d3b6408867 2024-11-21T00:28:55,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c97b2e23055b43c6b6e749281a4094f0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c97b2e23055b43c6b6e749281a4094f0 2024-11-21T00:28:55,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/7a7f430638b84f409f13d48f1b384d04 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/7a7f430638b84f409f13d48f1b384d04 2024-11-21T00:28:55,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9139fa6d44714023a87cbd179127b0ce to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9139fa6d44714023a87cbd179127b0ce 2024-11-21T00:28:55,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c1fab06f307f4d7392310f509d47a9ed to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c1fab06f307f4d7392310f509d47a9ed 2024-11-21T00:28:55,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/afd57d1e5182420fa06dc3799ed54700 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/afd57d1e5182420fa06dc3799ed54700 2024-11-21T00:28:55,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b9ebbb6982244c569c1aebd0d257cc9c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b9ebbb6982244c569c1aebd0d257cc9c 2024-11-21T00:28:55,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/659841b7c4674892bec338160b4258b4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/659841b7c4674892bec338160b4258b4 2024-11-21T00:28:55,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9ba3570c90674e4f91034420d5f0e18a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9ba3570c90674e4f91034420d5f0e18a 2024-11-21T00:28:55,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/a67761ac84a54727bb78c51f96709b64 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/a67761ac84a54727bb78c51f96709b64 2024-11-21T00:28:55,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/014919fc5a2b439e98b4e2b85222f061 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/014919fc5a2b439e98b4e2b85222f061 2024-11-21T00:28:55,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/273c2afe2e4341089a2ac7eab77cafd8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/273c2afe2e4341089a2ac7eab77cafd8 2024-11-21T00:28:55,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ef4f086726b1455d870def7bc7f98167 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ef4f086726b1455d870def7bc7f98167 2024-11-21T00:28:55,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/10d479665f5542a2b4a8e97ee9970e30 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/10d479665f5542a2b4a8e97ee9970e30 2024-11-21T00:28:55,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/33ca0b79d3f4431e98950601f6b7d4da to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/33ca0b79d3f4431e98950601f6b7d4da 2024-11-21T00:28:55,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f207c279c89049f68bab8fb191dd149b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f207c279c89049f68bab8fb191dd149b 2024-11-21T00:28:55,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/21109253651d492f89cf86f5a73c085c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/21109253651d492f89cf86f5a73c085c 2024-11-21T00:28:55,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/31a37b4b1a4b49b2a614b0d2673eac22 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/31a37b4b1a4b49b2a614b0d2673eac22 2024-11-21T00:28:55,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/12ff07f290fe4443a966f4a97e05e23a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/12ff07f290fe4443a966f4a97e05e23a 2024-11-21T00:28:55,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/2fa66121d7bc4479b553e4519f586ce8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/2fa66121d7bc4479b553e4519f586ce8 2024-11-21T00:28:55,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/6da0c93b1c894c0c85fccf49de6b7554 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/6da0c93b1c894c0c85fccf49de6b7554 2024-11-21T00:28:55,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ec89fe03a0754b71a63f493b16cfffd5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/ec89fe03a0754b71a63f493b16cfffd5 2024-11-21T00:28:55,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/bb19f4079651438b97bcf8d67425dcc8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/bb19f4079651438b97bcf8d67425dcc8 2024-11-21T00:28:55,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/8b1bbbf471424ca0b2db2b570b85c9a3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/8b1bbbf471424ca0b2db2b570b85c9a3 2024-11-21T00:28:55,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b0ab4f804b42497094741ac76c589674 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/b0ab4f804b42497094741ac76c589674 2024-11-21T00:28:55,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/05cba84911004b1ba69efaa5b060b99f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/05cba84911004b1ba69efaa5b060b99f 2024-11-21T00:28:55,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f296d5b14a874c1f8699b779442b5f77 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/f296d5b14a874c1f8699b779442b5f77 2024-11-21T00:28:55,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/3b8e7ad880664edb9bedef8f5b660f25 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/3b8e7ad880664edb9bedef8f5b660f25 2024-11-21T00:28:55,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c45f43fe2f784736b148405d16c7689b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/c45f43fe2f784736b148405d16c7689b 2024-11-21T00:28:55,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9abacd09336442168650d2b1529f6cf9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/9abacd09336442168650d2b1529f6cf9 2024-11-21T00:28:55,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4016b613ec1643ce9ab970f58b4c8958 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/4016b613ec1643ce9ab970f58b4c8958 2024-11-21T00:28:55,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/87add02bf35f4d6888e64fdb4142172c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/8e7762731f85471da1d51c98eab2d142, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b36e9b3edc9c4ea29463db1f7ce8ba66, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/cf930f2859aa4ae4b9f4b44f85db9191, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/749af46beb734dfaa757838fb8d08611, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/bf1b7f7f0451402fb482698f564f28ef, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/570e9510d7154c20929964a90486b386, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d0f59fc4866542adbf8ba0e2703bc6d9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/130906f046224cf092a7ddf7f3744c65, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1c7d13c8fcd64ae79bfef074800007d7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/576b88eceeca4c25baa8b2ba9f722957, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/3e67ddd332f347d8adc8e41a1837556a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e09a6d5009e74dc9ad1b266bd3830fd6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/fe03b7e133c74270ab40dea8fd2ab305, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1fb0ea7c604f4c2abe5b795893084943, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e6f05fc9f6de4174b9ced9d8f765b80e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/00f1decf22244231ab032261a42249c8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d1e0c15c64e345b1a9274d27e5e0b19d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ece4cb96e3a14957b80599f7d052878d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/988697bbc5b34252a48882fd2581af14, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/40f1fa75943749bf88651690867e7873, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/824b82b720ff462e82428f301f9dd9cc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d25491bbf8ac4fab9fc7fe871c558d20, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2f9b182fa35046d4bba65715f89f883e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2b0ae8e5308d480b8e94877451567f9e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/5e91509bfecb4523a1dbc5cb5fd76c4d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b7e3ce69db0442178713c3a1beaf6b69, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/05f6e4ef1cea4d90af574f516b587a2a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1a867553f2ab42449afe02524c1854f8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/c0fe38492df048b78760e2fa6bce251f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/659df8da369a4bef91090df1b7d2e4aa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ec42155fdcee4dffab5d9f2601a256ca, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/df99d88fac514ca98ac6c16feff2f925] to archive 2024-11-21T00:28:55,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:28:55,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/87add02bf35f4d6888e64fdb4142172c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/87add02bf35f4d6888e64fdb4142172c 2024-11-21T00:28:55,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/8e7762731f85471da1d51c98eab2d142 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/8e7762731f85471da1d51c98eab2d142 2024-11-21T00:28:55,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b36e9b3edc9c4ea29463db1f7ce8ba66 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b36e9b3edc9c4ea29463db1f7ce8ba66 2024-11-21T00:28:55,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/cf930f2859aa4ae4b9f4b44f85db9191 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/cf930f2859aa4ae4b9f4b44f85db9191 2024-11-21T00:28:55,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/749af46beb734dfaa757838fb8d08611 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/749af46beb734dfaa757838fb8d08611 2024-11-21T00:28:55,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/bf1b7f7f0451402fb482698f564f28ef to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/bf1b7f7f0451402fb482698f564f28ef 2024-11-21T00:28:55,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/570e9510d7154c20929964a90486b386 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/570e9510d7154c20929964a90486b386 2024-11-21T00:28:55,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d0f59fc4866542adbf8ba0e2703bc6d9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d0f59fc4866542adbf8ba0e2703bc6d9 2024-11-21T00:28:55,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/130906f046224cf092a7ddf7f3744c65 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/130906f046224cf092a7ddf7f3744c65 2024-11-21T00:28:55,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1c7d13c8fcd64ae79bfef074800007d7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1c7d13c8fcd64ae79bfef074800007d7 2024-11-21T00:28:55,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/576b88eceeca4c25baa8b2ba9f722957 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/576b88eceeca4c25baa8b2ba9f722957 2024-11-21T00:28:55,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/3e67ddd332f347d8adc8e41a1837556a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/3e67ddd332f347d8adc8e41a1837556a 2024-11-21T00:28:55,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e09a6d5009e74dc9ad1b266bd3830fd6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e09a6d5009e74dc9ad1b266bd3830fd6 2024-11-21T00:28:55,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/fe03b7e133c74270ab40dea8fd2ab305 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/fe03b7e133c74270ab40dea8fd2ab305 2024-11-21T00:28:55,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1fb0ea7c604f4c2abe5b795893084943 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1fb0ea7c604f4c2abe5b795893084943 2024-11-21T00:28:55,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e6f05fc9f6de4174b9ced9d8f765b80e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/e6f05fc9f6de4174b9ced9d8f765b80e 2024-11-21T00:28:55,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/00f1decf22244231ab032261a42249c8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/00f1decf22244231ab032261a42249c8 2024-11-21T00:28:55,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d1e0c15c64e345b1a9274d27e5e0b19d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d1e0c15c64e345b1a9274d27e5e0b19d 2024-11-21T00:28:55,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ece4cb96e3a14957b80599f7d052878d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ece4cb96e3a14957b80599f7d052878d 2024-11-21T00:28:55,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/988697bbc5b34252a48882fd2581af14 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/988697bbc5b34252a48882fd2581af14 2024-11-21T00:28:55,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/40f1fa75943749bf88651690867e7873 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/40f1fa75943749bf88651690867e7873 2024-11-21T00:28:55,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/824b82b720ff462e82428f301f9dd9cc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/824b82b720ff462e82428f301f9dd9cc 2024-11-21T00:28:55,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d25491bbf8ac4fab9fc7fe871c558d20 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/d25491bbf8ac4fab9fc7fe871c558d20 2024-11-21T00:28:55,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2f9b182fa35046d4bba65715f89f883e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2f9b182fa35046d4bba65715f89f883e 2024-11-21T00:28:55,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2b0ae8e5308d480b8e94877451567f9e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/2b0ae8e5308d480b8e94877451567f9e 2024-11-21T00:28:55,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/5e91509bfecb4523a1dbc5cb5fd76c4d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/5e91509bfecb4523a1dbc5cb5fd76c4d 2024-11-21T00:28:55,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b7e3ce69db0442178713c3a1beaf6b69 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/b7e3ce69db0442178713c3a1beaf6b69 2024-11-21T00:28:55,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/05f6e4ef1cea4d90af574f516b587a2a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/05f6e4ef1cea4d90af574f516b587a2a 2024-11-21T00:28:55,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1a867553f2ab42449afe02524c1854f8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/1a867553f2ab42449afe02524c1854f8 2024-11-21T00:28:55,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/c0fe38492df048b78760e2fa6bce251f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/c0fe38492df048b78760e2fa6bce251f 2024-11-21T00:28:55,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/659df8da369a4bef91090df1b7d2e4aa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/659df8da369a4bef91090df1b7d2e4aa 2024-11-21T00:28:55,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ec42155fdcee4dffab5d9f2601a256ca to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ec42155fdcee4dffab5d9f2601a256ca 2024-11-21T00:28:55,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/df99d88fac514ca98ac6c16feff2f925 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/df99d88fac514ca98ac6c16feff2f925 2024-11-21T00:28:55,385 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/recovered.edits/491.seqid, newMaxSeqId=491, maxSeqId=1 2024-11-21T00:28:55,385 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. 2024-11-21T00:28:55,385 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 5151f743f103348045956281ec3b4bb9: Waiting for close lock at 1732148933820Running coprocessor pre-close hooks at 1732148933820Disabling compacts and flushes for region at 1732148933820Disabling writes for close at 1732148933820Obtaining lock to block concurrent updates at 1732148933820Preparing flush snapshotting stores in 5151f743f103348045956281ec3b4bb9 at 1732148933820Finished memstore snapshotting TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732148933820Flushing stores of TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9. at 1732148933821 (+1 ms)Flushing 5151f743f103348045956281ec3b4bb9/A: creating writer at 1732148933821Flushing 5151f743f103348045956281ec3b4bb9/A: appending metadata at 1732148933826 (+5 ms)Flushing 5151f743f103348045956281ec3b4bb9/A: closing flushed file at 1732148933826Flushing 5151f743f103348045956281ec3b4bb9/B: creating writer at 1732148934260 (+434 ms)Flushing 5151f743f103348045956281ec3b4bb9/B: appending metadata at 1732148934269 (+9 ms)Flushing 5151f743f103348045956281ec3b4bb9/B: closing flushed file at 1732148934269Flushing 5151f743f103348045956281ec3b4bb9/C: creating writer at 1732148934691 (+422 ms)Flushing 5151f743f103348045956281ec3b4bb9/C: appending metadata at 1732148934698 (+7 ms)Flushing 5151f743f103348045956281ec3b4bb9/C: closing flushed file at 1732148934698Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e7fcf19: reopening flushed file at 1732148935145 (+447 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34222851: reopening flushed file at 1732148935153 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7602701f: reopening flushed file at 1732148935158 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5151f743f103348045956281ec3b4bb9 in 1349ms, sequenceid=488, compaction requested=true at 1732148935169 (+11 ms)Writing region close event to WAL at 1732148935382 (+213 ms)Running coprocessor post-close hooks at 1732148935385 (+3 ms)Closed at 1732148935385 2024-11-21T00:28:55,387 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:55,388 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=5151f743f103348045956281ec3b4bb9, regionState=CLOSED 2024-11-21T00:28:55,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:28:55,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T00:28:55,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 5151f743f103348045956281ec3b4bb9, server=5ed4808ef0e6,38589,1732148778087 in 1.7240 sec 2024-11-21T00:28:55,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-21T00:28:55,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5151f743f103348045956281ec3b4bb9, UNASSIGN in 1.7290 sec 2024-11-21T00:28:55,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T00:28:55,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7330 sec 2024-11-21T00:28:55,396 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148935396"}]},"ts":"1732148935396"} 2024-11-21T00:28:55,398 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:28:55,398 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:28:55,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7440 sec 2024-11-21T00:28:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T00:28:55,802 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:55,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:28:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,804 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T00:28:55,805 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,806 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:55,808 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/recovered.edits] 2024-11-21T00:28:55,809 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1bd5a451346047558f781f6607914dd8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/1bd5a451346047558f781f6607914dd8 2024-11-21T00:28:55,810 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/5e0b0d4334714940b5c574a98570eab3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/5e0b0d4334714940b5c574a98570eab3 2024-11-21T00:28:55,811 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f22ed81ca6754f279ab83234ad6a9062 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/A/f22ed81ca6754f279ab83234ad6a9062 2024-11-21T00:28:55,812 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/1c1cddccd63b492cb216fcb6b8963a42 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/1c1cddccd63b492cb216fcb6b8963a42 2024-11-21T00:28:55,813 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/488c4aebb9794f2c8f254111e8bec1ea to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/488c4aebb9794f2c8f254111e8bec1ea 2024-11-21T00:28:55,814 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/86994df8f5e24991ab7507a65be8c88c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/B/86994df8f5e24991ab7507a65be8c88c 2024-11-21T00:28:55,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/104f70a6516f4716991291c55bcfdfde to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/104f70a6516f4716991291c55bcfdfde 2024-11-21T00:28:55,817 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/75a1822890fa415f8cab655be4dfcdf7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/75a1822890fa415f8cab655be4dfcdf7 2024-11-21T00:28:55,818 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ff6cb3d2e0474a68ae618697f633f1aa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/C/ff6cb3d2e0474a68ae618697f633f1aa 2024-11-21T00:28:55,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/recovered.edits/491.seqid to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9/recovered.edits/491.seqid 2024-11-21T00:28:55,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/5151f743f103348045956281ec3b4bb9 2024-11-21T00:28:55,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:28:55,823 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,825 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:28:55,827 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:28:55,828 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,828 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:28:55,828 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148935828"}]},"ts":"9223372036854775807"} 2024-11-21T00:28:55,830 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:28:55,830 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5151f743f103348045956281ec3b4bb9, NAME => 'TestAcidGuarantees,,1732148903725.5151f743f103348045956281ec3b4bb9.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:28:55,830 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:28:55,830 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148935830"}]},"ts":"9223372036854775807"} 2024-11-21T00:28:55,832 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:28:55,841 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-21T00:28:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T00:28:55,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:28:55,912 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:55,922 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 240), OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1081 (was 940) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2131 (was 3032) 2024-11-21T00:28:55,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1081, ProcessCount=11, AvailableMemoryMB=2131 2024-11-21T00:28:55,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:28:55,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:28:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:55,935 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T00:28:55,935 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:55,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-21T00:28:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:28:55,936 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T00:28:55,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742410_1586 (size=959) 2024-11-21T00:28:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:28:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:28:56,349 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29 2024-11-21T00:28:56,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742411_1587 (size=53) 2024-11-21T00:28:56,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:28:56,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 619ceef852c1cdfc78ce20d02329c56f, disabling compactions & flushes 2024-11-21T00:28:56,360 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. after waiting 0 ms 2024-11-21T00:28:56,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,360 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 619ceef852c1cdfc78ce20d02329c56f: Waiting for close lock at 1732148936360Disabling compacts and flushes for region at 1732148936360Disabling writes for close at 1732148936360Writing region close event to WAL at 1732148936360Closed at 1732148936360 2024-11-21T00:28:56,361 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T00:28:56,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732148936361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732148936361"}]},"ts":"1732148936361"} 2024-11-21T00:28:56,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T00:28:56,364 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T00:28:56,364 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148936364"}]},"ts":"1732148936364"} 2024-11-21T00:28:56,365 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T00:28:56,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, ASSIGN}] 2024-11-21T00:28:56,366 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, ASSIGN 2024-11-21T00:28:56,367 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, ASSIGN; state=OFFLINE, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=false 2024-11-21T00:28:56,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:56,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, ASSIGN because future has completed 2024-11-21T00:28:56,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:28:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:28:56,674 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,675 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:28:56,675 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,675 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:28:56,675 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,675 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,676 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,678 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619ceef852c1cdfc78ce20d02329c56f columnFamilyName A 2024-11-21T00:28:56,678 DEBUG [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:56,678 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(327): Store=619ceef852c1cdfc78ce20d02329c56f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:56,678 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,679 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619ceef852c1cdfc78ce20d02329c56f columnFamilyName B 2024-11-21T00:28:56,679 DEBUG [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:56,680 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(327): Store=619ceef852c1cdfc78ce20d02329c56f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:56,680 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,680 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619ceef852c1cdfc78ce20d02329c56f columnFamilyName C 2024-11-21T00:28:56,681 DEBUG [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:56,681 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(327): Store=619ceef852c1cdfc78ce20d02329c56f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:56,681 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,681 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,682 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,682 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,682 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,690 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:28:56,694 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,696 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T00:28:56,697 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 619ceef852c1cdfc78ce20d02329c56f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66152463, jitterRate=-0.014251485466957092}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:28:56,697 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:56,697 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 619ceef852c1cdfc78ce20d02329c56f: Running coprocessor pre-open hook at 1732148936675Writing region info on filesystem at 1732148936675Initializing all the Stores at 1732148936676 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148936676Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148936676Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148936676Cleaning up temporary data from old regions at 1732148936682 (+6 ms)Running coprocessor post-open hooks at 1732148936697 (+15 ms)Region opened successfully at 1732148936697 2024-11-21T00:28:56,698 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., pid=142, masterSystemTime=1732148936671 2024-11-21T00:28:56,701 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,701 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:56,702 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=OPEN, openSeqNum=2, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:56,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:28:56,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T00:28:56,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 in 184 msec 2024-11-21T00:28:56,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-21T00:28:56,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, ASSIGN in 341 msec 2024-11-21T00:28:56,712 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T00:28:56,712 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148936712"}]},"ts":"1732148936712"} 2024-11-21T00:28:56,714 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T00:28:56,715 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T00:28:56,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 782 msec 2024-11-21T00:28:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-21T00:28:57,072 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:57,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b31ef7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:57,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:57,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:57,074 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133d5ae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:57,076 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e56a8bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:57,078 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:57,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:57,081 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T00:28:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 5ed4808ef0e6,38537,1732148777004 2024-11-21T00:28:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6acde34c 2024-11-21T00:28:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T00:28:57,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T00:28:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T00:28:57,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T00:28:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T00:28:57,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742412_1588 (size=995) 2024-11-21T00:28:57,553 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T00:28:57,553 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T00:28:57,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:28:57,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, REOPEN/MOVE}] 2024-11-21T00:28:57,563 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, REOPEN/MOVE 2024-11-21T00:28:57,567 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:57,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, REOPEN/MOVE because future has completed 2024-11-21T00:28:57,569 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:28:57,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:28:57,722 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:57,722 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:28:57,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 619ceef852c1cdfc78ce20d02329c56f, disabling compactions & flushes 2024-11-21T00:28:57,723 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:57,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:57,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. after waiting 0 ms 2024-11-21T00:28:57,723 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:57,746 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T00:28:57,747 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:57,747 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 619ceef852c1cdfc78ce20d02329c56f: Waiting for close lock at 1732148937723Running coprocessor pre-close hooks at 1732148937723Disabling compacts and flushes for region at 1732148937723Disabling writes for close at 1732148937723Writing region close event to WAL at 1732148937740 (+17 ms)Running coprocessor post-close hooks at 1732148937747 (+7 ms)Closed at 1732148937747 2024-11-21T00:28:57,747 WARN [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 619ceef852c1cdfc78ce20d02329c56f to self. 2024-11-21T00:28:57,751 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:57,757 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=CLOSED 2024-11-21T00:28:57,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:28:57,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-21T00:28:57,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 in 192 msec 2024-11-21T00:28:57,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, REOPEN/MOVE; state=CLOSED, location=5ed4808ef0e6,38589,1732148778087; forceNewPlan=false, retain=true 2024-11-21T00:28:57,915 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=OPENING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:57,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, REOPEN/MOVE because future has completed 2024-11-21T00:28:57,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:28:58,086 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,087 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} 2024-11-21T00:28:58,087 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,087 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T00:28:58,087 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,087 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,105 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,109 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619ceef852c1cdfc78ce20d02329c56f columnFamilyName A 2024-11-21T00:28:58,120 DEBUG [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:58,124 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(327): Store=619ceef852c1cdfc78ce20d02329c56f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:58,125 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,132 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619ceef852c1cdfc78ce20d02329c56f columnFamilyName B 2024-11-21T00:28:58,132 DEBUG [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:58,133 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(327): Store=619ceef852c1cdfc78ce20d02329c56f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:58,134 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,136 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619ceef852c1cdfc78ce20d02329c56f columnFamilyName C 2024-11-21T00:28:58,136 DEBUG [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:58,136 INFO [StoreOpener-619ceef852c1cdfc78ce20d02329c56f-1 {}] regionserver.HStore(327): Store=619ceef852c1cdfc78ce20d02329c56f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T00:28:58,136 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,137 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,138 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,138 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,138 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,139 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T00:28:58,140 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,140 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 619ceef852c1cdfc78ce20d02329c56f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60184858, jitterRate=-0.10317572951316833}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T00:28:58,140 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,144 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 619ceef852c1cdfc78ce20d02329c56f: Running coprocessor pre-open hook at 1732148938087Writing region info on filesystem at 1732148938087Initializing all the Stores at 1732148938104 (+17 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148938104Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148938105 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732148938105Cleaning up temporary data from old regions at 1732148938138 (+33 ms)Running coprocessor post-open hooks at 1732148938140 (+2 ms)Region opened successfully at 1732148938141 (+1 ms) 2024-11-21T00:28:58,148 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., pid=147, masterSystemTime=1732148938082 2024-11-21T00:28:58,157 DEBUG [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,157 INFO [RS_OPEN_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,157 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=OPEN, openSeqNum=5, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:28:58,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-21T00:28:58,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 in 244 msec 2024-11-21T00:28:58,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-21T00:28:58,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, REOPEN/MOVE in 621 msec 2024-11-21T00:28:58,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T00:28:58,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 631 msec 2024-11-21T00:28:58,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1110 sec 2024-11-21T00:28:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T00:28:58,200 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:28:58,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,205 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1e2df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f59ae31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,218 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8c4430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,219 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e996545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694d52df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,231 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11f3c31a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51794eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1394f730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,241 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eded322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20743e1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333a15b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36960429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53a4695f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,255 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3299cf17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,261 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dca1b7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1598bed5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,271 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10db8b77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5ed4808ef0e6,38537,-1 for getting cluster id 2024-11-21T00:28:58,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T00:28:58,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f9997a-520c-400d-b987-9988e298295c' 2024-11-21T00:28:58,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T00:28:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f9997a-520c-400d-b987-9988e298295c" 2024-11-21T00:28:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363e22f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5ed4808ef0e6,38537,-1] 2024-11-21T00:28:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T00:28:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:28:58,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T00:28:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52aebc78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T00:28:58,276 DEBUG [Thread-2617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,280 DEBUG [Thread-2602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,289 DEBUG [Thread-2608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,292 DEBUG [Thread-2615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,294 DEBUG [Thread-2615 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,294 DEBUG [Thread-2615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,297 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:28:58,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:28:58,308 DEBUG [Thread-2604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,316 DEBUG [Thread-2613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,326 DEBUG [Thread-2619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,327 DEBUG [Thread-2600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,336 DEBUG [Thread-2606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,347 DEBUG [Thread-2611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T00:28:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5ed4808ef0e6,38589,1732148778087, seqNum=-1] 2024-11-21T00:28:58,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T00:28:58,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T00:28:58,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:28:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5] 2024-11-21T00:28:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-21T00:28:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:28:58,358 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:28:58,358 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:28:58,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:28:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148998348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148998350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148998361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148998370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732148998372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b9b68bdaf2204e0eb9eb8ff8eea399b9_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148938293/Put/seqid=0 2024-11-21T00:28:58,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742413_1589 (size=12154) 2024-11-21T00:28:58,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:58,439 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b9b68bdaf2204e0eb9eb8ff8eea399b9_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b9b68bdaf2204e0eb9eb8ff8eea399b9_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:58,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/492c8685a5574f8498fbecdfb677139b, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:28:58,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/492c8685a5574f8498fbecdfb677139b is 175, key is test_row_0/A:col10/1732148938293/Put/seqid=0 2024-11-21T00:28:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:28:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742414_1590 (size=30955) 2024-11-21T00:28:58,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/492c8685a5574f8498fbecdfb677139b 2024-11-21T00:28:58,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148998472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148998472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148998483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148998483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732148998483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/c43254b728d148f9a3dff0e5e020e0ed is 50, key is test_row_0/B:col10/1732148938293/Put/seqid=0 2024-11-21T00:28:58,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:28:58,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:28:58,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742415_1591 (size=12001) 2024-11-21T00:28:58,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/c43254b728d148f9a3dff0e5e020e0ed 2024-11-21T00:28:58,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/2f8b7f3b961a477d8e61021a000c14dd is 50, key is test_row_0/C:col10/1732148938293/Put/seqid=0 2024-11-21T00:28:58,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742416_1592 (size=12001) 2024-11-21T00:28:58,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:28:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:28:58,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:28:58,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148998682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148998682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148998692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148998692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732148998702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:58,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:28:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:28:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:28:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:28:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:58,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:28:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148998993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148998993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148999002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148999002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732148999014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/2f8b7f3b961a477d8e61021a000c14dd 2024-11-21T00:28:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/492c8685a5574f8498fbecdfb677139b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b 2024-11-21T00:28:59,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b, entries=150, sequenceid=15, filesize=30.2 K 2024-11-21T00:28:59,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/c43254b728d148f9a3dff0e5e020e0ed as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/c43254b728d148f9a3dff0e5e020e0ed 2024-11-21T00:28:59,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/c43254b728d148f9a3dff0e5e020e0ed, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T00:28:59,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:28:59,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:59,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:28:59,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:59,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:59,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:59,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/2f8b7f3b961a477d8e61021a000c14dd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f8b7f3b961a477d8e61021a000c14dd 2024-11-21T00:28:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:28:59,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f8b7f3b961a477d8e61021a000c14dd, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T00:28:59,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 619ceef852c1cdfc78ce20d02329c56f in 876ms, sequenceid=15, compaction requested=false 2024-11-21T00:28:59,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:28:59,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T00:28:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:59,298 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:28:59,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121939ec99f17654658acbb6574f153a517_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148938350/Put/seqid=0 2024-11-21T00:28:59,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742417_1593 (size=12154) 2024-11-21T00:28:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:28:59,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:28:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:59,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148999529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732148999532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148999533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148999538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148999543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148999657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148999657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148999657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148999658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:28:59,743 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121939ec99f17654658acbb6574f153a517_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121939ec99f17654658acbb6574f153a517_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:28:59,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6237c17453be4bdeba144eccd33706f6, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:28:59,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6237c17453be4bdeba144eccd33706f6 is 175, key is test_row_0/A:col10/1732148938350/Put/seqid=0 2024-11-21T00:28:59,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742418_1594 (size=30955) 2024-11-21T00:28:59,798 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6237c17453be4bdeba144eccd33706f6 2024-11-21T00:28:59,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/dddf0562bba1420c969be57586fd58d1 is 50, key is test_row_0/B:col10/1732148938350/Put/seqid=0 2024-11-21T00:28:59,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742419_1595 (size=12001) 2024-11-21T00:28:59,867 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/dddf0562bba1420c969be57586fd58d1 2024-11-21T00:28:59,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732148999874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732148999874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732148999874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:28:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732148999878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:28:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:28:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:28:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/c8683b2fbb794a0ea470b1cc46fbb44c is 50, key is test_row_0/C:col10/1732148938350/Put/seqid=0 2024-11-21T00:28:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742420_1596 (size=12001) 2024-11-21T00:28:59,913 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/c8683b2fbb794a0ea470b1cc46fbb44c 2024-11-21T00:28:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6237c17453be4bdeba144eccd33706f6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6 2024-11-21T00:28:59,933 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6, entries=150, sequenceid=42, filesize=30.2 K 2024-11-21T00:28:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/dddf0562bba1420c969be57586fd58d1 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/dddf0562bba1420c969be57586fd58d1 2024-11-21T00:28:59,957 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/dddf0562bba1420c969be57586fd58d1, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T00:28:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/c8683b2fbb794a0ea470b1cc46fbb44c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c8683b2fbb794a0ea470b1cc46fbb44c 2024-11-21T00:28:59,962 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c8683b2fbb794a0ea470b1cc46fbb44c, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T00:28:59,963 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 619ceef852c1cdfc78ce20d02329c56f in 665ms, sequenceid=42, compaction requested=false 2024-11-21T00:28:59,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:28:59,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:28:59,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-21T00:28:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-21T00:28:59,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-21T00:28:59,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-11-21T00:28:59,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.6170 sec 2024-11-21T00:29:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:00,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:00,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:29:00,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214ad45e710151418d952f3bb357318a86_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:00,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149000220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149000222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742421_1597 (size=14594) 2024-11-21T00:29:00,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149000225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149000225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214ad45e710151418d952f3bb357318a86_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214ad45e710151418d952f3bb357318a86_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:00,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b7c7b103476e41b2b1966f58c090d4c0, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:00,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b7c7b103476e41b2b1966f58c090d4c0 is 175, key is test_row_0/A:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:00,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742422_1598 (size=39549) 2024-11-21T00:29:00,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149000332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149000332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149000342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149000343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T00:29:00,516 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:00,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-21T00:29:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:29:00,519 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:00,520 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:00,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149000542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149000544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149000552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149000552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732149000562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,564 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:29:00,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b7c7b103476e41b2b1966f58c090d4c0 2024-11-21T00:29:00,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/9a847013cb4e4fd1832b587742b4eacd is 50, key is test_row_0/B:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742423_1599 (size=12001) 2024-11-21T00:29:00,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:29:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:00,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:29:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:00,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:29:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149000855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149000857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149000863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149000864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:00,911 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:29:00,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:29:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:00,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:01,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/9a847013cb4e4fd1832b587742b4eacd 2024-11-21T00:29:01,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ea62425569f94ead8adb0186971de36a is 50, key is test_row_0/C:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:01,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742424_1600 (size=12001) 2024-11-21T00:29:01,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ea62425569f94ead8adb0186971de36a 2024-11-21T00:29:01,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b7c7b103476e41b2b1966f58c090d4c0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0 2024-11-21T00:29:01,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0, entries=200, sequenceid=53, filesize=38.6 K 2024-11-21T00:29:01,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/9a847013cb4e4fd1832b587742b4eacd as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9a847013cb4e4fd1832b587742b4eacd 2024-11-21T00:29:01,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9a847013cb4e4fd1832b587742b4eacd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T00:29:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ea62425569f94ead8adb0186971de36a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ea62425569f94ead8adb0186971de36a 2024-11-21T00:29:01,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ea62425569f94ead8adb0186971de36a, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T00:29:01,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 619ceef852c1cdfc78ce20d02329c56f in 951ms, sequenceid=53, compaction requested=true 2024-11-21T00:29:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:01,137 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:01,137 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:01,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:01,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:01,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T00:29:01,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:01,139 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:29:01,140 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:01,140 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:01,140 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:01,140 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=99.1 K 2024-11-21T00:29:01,140 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:01,140 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0] 2024-11-21T00:29:01,141 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:01,141 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:01,141 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:01,141 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/c43254b728d148f9a3dff0e5e020e0ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/dddf0562bba1420c969be57586fd58d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9a847013cb4e4fd1832b587742b4eacd] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=35.2 K 2024-11-21T00:29:01,142 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c43254b728d148f9a3dff0e5e020e0ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148938293 2024-11-21T00:29:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:29:01,142 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492c8685a5574f8498fbecdfb677139b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148938293 2024-11-21T00:29:01,143 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting dddf0562bba1420c969be57586fd58d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732148938348 2024-11-21T00:29:01,143 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6237c17453be4bdeba144eccd33706f6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732148938348 2024-11-21T00:29:01,143 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a847013cb4e4fd1832b587742b4eacd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148939532 2024-11-21T00:29:01,144 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c7b103476e41b2b1966f58c090d4c0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148939510 2024-11-21T00:29:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210bc6ddd1d4174c8bb58b01c4648df0b2_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148940223/Put/seqid=0 2024-11-21T00:29:01,170 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:01,170 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ecc1831e0c9e4c02a006050832e0d2bb is 50, key is test_row_0/B:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:01,181 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:01,213 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a95e05d011aa4a99a50940a8593dfe04_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:01,215 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a95e05d011aa4a99a50940a8593dfe04_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:01,215 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a95e05d011aa4a99a50940a8593dfe04_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:01,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742425_1601 (size=12154) 2024-11-21T00:29:01,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742426_1602 (size=12104) 2024-11-21T00:29:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:01,237 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210bc6ddd1d4174c8bb58b01c4648df0b2_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210bc6ddd1d4174c8bb58b01c4648df0b2_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1e3a0736fcbb4df1915b49183f1d1574, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:01,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1e3a0736fcbb4df1915b49183f1d1574 is 175, key is test_row_0/A:col10/1732148940223/Put/seqid=0 2024-11-21T00:29:01,240 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ecc1831e0c9e4c02a006050832e0d2bb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ecc1831e0c9e4c02a006050832e0d2bb 2024-11-21T00:29:01,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742427_1603 (size=4469) 2024-11-21T00:29:01,253 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#511 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:01,254 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/8ac130e21deb441f9ba4ed87a398e280 is 175, key is test_row_0/A:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:01,254 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into ecc1831e0c9e4c02a006050832e0d2bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:01,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:01,254 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=13, startTime=1732148941137; duration=0sec 2024-11-21T00:29:01,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:01,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:01,254 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:01,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742429_1605 (size=31058) 2024-11-21T00:29:01,277 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:01,277 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:01,277 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:01,277 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f8b7f3b961a477d8e61021a000c14dd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c8683b2fbb794a0ea470b1cc46fbb44c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ea62425569f94ead8adb0186971de36a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=35.2 K 2024-11-21T00:29:01,278 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f8b7f3b961a477d8e61021a000c14dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732148938293 2024-11-21T00:29:01,278 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c8683b2fbb794a0ea470b1cc46fbb44c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732148938348 2024-11-21T00:29:01,279 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ea62425569f94ead8adb0186971de36a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148939532 2024-11-21T00:29:01,282 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/8ac130e21deb441f9ba4ed87a398e280 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/8ac130e21deb441f9ba4ed87a398e280 2024-11-21T00:29:01,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742428_1604 (size=30955) 2024-11-21T00:29:01,289 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1e3a0736fcbb4df1915b49183f1d1574 2024-11-21T00:29:01,289 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into 8ac130e21deb441f9ba4ed87a398e280(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:01,290 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:01,290 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=13, startTime=1732148941137; duration=0sec 2024-11-21T00:29:01,290 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:01,290 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:01,305 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:01,306 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/13f95eeb521743e38c80f39c843fb4de is 50, key is test_row_0/C:col10/1732148939532/Put/seqid=0 2024-11-21T00:29:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5ed79fa476d34176b1b0a9a3bfae86cc is 50, key is test_row_0/B:col10/1732148940223/Put/seqid=0 2024-11-21T00:29:01,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742430_1606 (size=12104) 2024-11-21T00:29:01,362 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/13f95eeb521743e38c80f39c843fb4de as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/13f95eeb521743e38c80f39c843fb4de 2024-11-21T00:29:01,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742431_1607 (size=12001) 2024-11-21T00:29:01,367 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into 13f95eeb521743e38c80f39c843fb4de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:01,367 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:01,367 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=13, startTime=1732148941137; duration=0sec 2024-11-21T00:29:01,368 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:01,368 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:01,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149001388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149001388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149001391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149001394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149001502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149001502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149001513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:29:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149001712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149001713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149001722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:01,755 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5ed79fa476d34176b1b0a9a3bfae86cc 2024-11-21T00:29:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bfb666d59e804618902c72125ae23a7f is 50, key is test_row_0/C:col10/1732148940223/Put/seqid=0 2024-11-21T00:29:01,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742432_1608 (size=12001) 2024-11-21T00:29:02,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149002022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149002022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149002032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,206 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bfb666d59e804618902c72125ae23a7f 2024-11-21T00:29:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1e3a0736fcbb4df1915b49183f1d1574 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574 2024-11-21T00:29:02,214 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574, entries=150, sequenceid=79, filesize=30.2 K 2024-11-21T00:29:02,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5ed79fa476d34176b1b0a9a3bfae86cc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5ed79fa476d34176b1b0a9a3bfae86cc 2024-11-21T00:29:02,218 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5ed79fa476d34176b1b0a9a3bfae86cc, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T00:29:02,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bfb666d59e804618902c72125ae23a7f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bfb666d59e804618902c72125ae23a7f 2024-11-21T00:29:02,228 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bfb666d59e804618902c72125ae23a7f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-21T00:29:02,231 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 619ceef852c1cdfc78ce20d02329c56f in 1092ms, sequenceid=79, compaction requested=false 2024-11-21T00:29:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-21T00:29:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-21T00:29:02,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T00:29:02,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7120 sec 2024-11-21T00:29:02,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.7190 sec 2024-11-21T00:29:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:02,404 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:02,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:29:02,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214b963f398d3c431f9c64e2ea4eb55b11_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:02,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742433_1609 (size=14594) 2024-11-21T00:29:02,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:02,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214b963f398d3c431f9c64e2ea4eb55b11_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214b963f398d3c431f9c64e2ea4eb55b11_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:02,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d47b8e52d6354ca883de6c85e51e058c, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:02,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d47b8e52d6354ca883de6c85e51e058c is 175, key is test_row_0/A:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:02,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742434_1610 (size=39549) 2024-11-21T00:29:02,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149002525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149002532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149002532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149002542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732149002572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,575 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149002632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T00:29:02,662 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:02,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-21T00:29:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:02,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:02,666 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:02,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:02,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:29:02,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:02,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:02,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:02,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:02,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149002842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:02,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d47b8e52d6354ca883de6c85e51e058c 2024-11-21T00:29:02,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/45120c6b64ea4124b58ae8b665d9ac4a is 50, key is test_row_0/B:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:02,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742435_1611 (size=12001) 2024-11-21T00:29:02,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:29:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:02,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:29:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149003152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:03,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:29:03,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:03,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/45120c6b64ea4124b58ae8b665d9ac4a 2024-11-21T00:29:03,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/da53ef427c4d421dbbffee944e84af31 is 50, key is test_row_0/C:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:03,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742436_1612 (size=12001) 2024-11-21T00:29:03,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/da53ef427c4d421dbbffee944e84af31 2024-11-21T00:29:03,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:29:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:03,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d47b8e52d6354ca883de6c85e51e058c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c 2024-11-21T00:29:03,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c, entries=200, sequenceid=93, filesize=38.6 K 2024-11-21T00:29:03,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/45120c6b64ea4124b58ae8b665d9ac4a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/45120c6b64ea4124b58ae8b665d9ac4a 2024-11-21T00:29:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149003542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:03,546 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:03,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/45120c6b64ea4124b58ae8b665d9ac4a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T00:29:03,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/da53ef427c4d421dbbffee944e84af31 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/da53ef427c4d421dbbffee944e84af31 2024-11-21T00:29:03,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149003552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149003553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:03,566 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:03,566 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:03,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/da53ef427c4d421dbbffee944e84af31, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T00:29:03,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 619ceef852c1cdfc78ce20d02329c56f in 1197ms, sequenceid=93, compaction requested=true 2024-11-21T00:29:03,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:03,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:03,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:03,602 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:03,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:03,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:03,602 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:03,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:03,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:03,608 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:03,608 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:03,608 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:03,608 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:03,608 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,608 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,609 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ecc1831e0c9e4c02a006050832e0d2bb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5ed79fa476d34176b1b0a9a3bfae86cc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/45120c6b64ea4124b58ae8b665d9ac4a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=35.3 K 2024-11-21T00:29:03,609 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/8ac130e21deb441f9ba4ed87a398e280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=99.2 K 2024-11-21T00:29:03,609 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,609 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/8ac130e21deb441f9ba4ed87a398e280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c] 2024-11-21T00:29:03,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T00:29:03,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,610 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:29:03,613 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc1831e0c9e4c02a006050832e0d2bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148939532 2024-11-21T00:29:03,616 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac130e21deb441f9ba4ed87a398e280, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148939532 2024-11-21T00:29:03,616 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed79fa476d34176b1b0a9a3bfae86cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732148940221 2024-11-21T00:29:03,617 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 45120c6b64ea4124b58ae8b665d9ac4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732148941376 2024-11-21T00:29:03,620 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e3a0736fcbb4df1915b49183f1d1574, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732148940221 2024-11-21T00:29:03,620 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47b8e52d6354ca883de6c85e51e058c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732148941376 2024-11-21T00:29:03,630 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:03,630 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b6255816e41843ba8156eab990c1e2fe is 50, key is test_row_0/B:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:03,637 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:03,657 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211576b05f09494b83a37743e54338de95_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:03,659 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211576b05f09494b83a37743e54338de95_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:03,659 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211576b05f09494b83a37743e54338de95_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:03,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742437_1613 (size=12207) 2024-11-21T00:29:03,668 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b6255816e41843ba8156eab990c1e2fe as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b6255816e41843ba8156eab990c1e2fe 2024-11-21T00:29:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:03,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:03,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121980c06f3f1a9450cbd5074a10b310974_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148942519/Put/seqid=0 2024-11-21T00:29:03,688 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into b6255816e41843ba8156eab990c1e2fe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:03,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:03,689 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=13, startTime=1732148943602; duration=0sec 2024-11-21T00:29:03,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:03,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:03,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:03,691 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:03,691 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:03,691 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:03,691 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/13f95eeb521743e38c80f39c843fb4de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bfb666d59e804618902c72125ae23a7f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/da53ef427c4d421dbbffee944e84af31] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=35.3 K 2024-11-21T00:29:03,691 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 13f95eeb521743e38c80f39c843fb4de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732148939532 2024-11-21T00:29:03,692 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb666d59e804618902c72125ae23a7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732148940221 2024-11-21T00:29:03,695 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting da53ef427c4d421dbbffee944e84af31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732148941376 2024-11-21T00:29:03,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149003725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:03,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742438_1614 (size=4469) 2024-11-21T00:29:03,740 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:03,741 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/80cc7901a43d47a38921858a3d2c2382 is 50, key is test_row_0/C:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:03,743 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#519 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:03,743 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/3c1a0e53e9bb40cc87eb70ada5052a3a is 175, key is test_row_0/A:col10/1732148942403/Put/seqid=0 2024-11-21T00:29:03,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742439_1615 (size=12154) 2024-11-21T00:29:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:03,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742440_1616 (size=12207) 2024-11-21T00:29:03,844 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/80cc7901a43d47a38921858a3d2c2382 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/80cc7901a43d47a38921858a3d2c2382 2024-11-21T00:29:03,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149003843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742441_1617 (size=31161) 2024-11-21T00:29:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:03,865 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into 80cc7901a43d47a38921858a3d2c2382(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:03,865 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:03,865 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=13, startTime=1732148943602; duration=0sec 2024-11-21T00:29:03,865 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:03,865 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:04,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149004064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:04,189 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121980c06f3f1a9450cbd5074a10b310974_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121980c06f3f1a9450cbd5074a10b310974_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/cb580719d7c849a9abe4d981de5e68f7, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/cb580719d7c849a9abe4d981de5e68f7 is 175, key is test_row_0/A:col10/1732148942519/Put/seqid=0 2024-11-21T00:29:04,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742442_1618 (size=30955) 2024-11-21T00:29:04,232 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/cb580719d7c849a9abe4d981de5e68f7 2024-11-21T00:29:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5dfa59d1b1604f2e828d9debdac4d577 is 50, key is test_row_0/B:col10/1732148942519/Put/seqid=0 2024-11-21T00:29:04,284 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/3c1a0e53e9bb40cc87eb70ada5052a3a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/3c1a0e53e9bb40cc87eb70ada5052a3a 2024-11-21T00:29:04,302 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into 3c1a0e53e9bb40cc87eb70ada5052a3a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:04,302 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:04,303 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=13, startTime=1732148943601; duration=0sec 2024-11-21T00:29:04,303 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:04,303 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742443_1619 (size=12001) 2024-11-21T00:29:04,312 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5dfa59d1b1604f2e828d9debdac4d577 2024-11-21T00:29:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/418fa2266148430fb520a55a4c6a70cc is 50, key is test_row_0/C:col10/1732148942519/Put/seqid=0 2024-11-21T00:29:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149004372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:04,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742444_1620 (size=12001) 2024-11-21T00:29:04,814 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/418fa2266148430fb520a55a4c6a70cc 2024-11-21T00:29:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/cb580719d7c849a9abe4d981de5e68f7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7 2024-11-21T00:29:04,828 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7, entries=150, sequenceid=118, filesize=30.2 K 2024-11-21T00:29:04,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5dfa59d1b1604f2e828d9debdac4d577 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5dfa59d1b1604f2e828d9debdac4d577 2024-11-21T00:29:04,835 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5dfa59d1b1604f2e828d9debdac4d577, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:29:04,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/418fa2266148430fb520a55a4c6a70cc as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/418fa2266148430fb520a55a4c6a70cc 2024-11-21T00:29:04,842 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/418fa2266148430fb520a55a4c6a70cc, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T00:29:04,845 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 619ceef852c1cdfc78ce20d02329c56f in 1234ms, sequenceid=118, compaction requested=false 2024-11-21T00:29:04,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:04,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:04,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-21T00:29:04,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-21T00:29:04,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-21T00:29:04,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-21T00:29:04,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1850 sec 2024-11-21T00:29:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:04,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T00:29:04,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b82e14efade8428ba3cc57ed02f9e199_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742445_1621 (size=14694) 2024-11-21T00:29:04,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:04,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b82e14efade8428ba3cc57ed02f9e199_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b82e14efade8428ba3cc57ed02f9e199_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:04,989 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/f6693a1b0c2746f5a019d96bb82fba81, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:04,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/f6693a1b0c2746f5a019d96bb82fba81 is 175, key is test_row_0/A:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:05,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742446_1622 (size=39649) 2024-11-21T00:29:05,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149005031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149005142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149005352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/f6693a1b0c2746f5a019d96bb82fba81 2024-11-21T00:29:05,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/514578f2b5c340edae6b6b7cfc2df391 is 50, key is test_row_0/B:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:05,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742447_1623 (size=12101) 2024-11-21T00:29:05,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/514578f2b5c340edae6b6b7cfc2df391 2024-11-21T00:29:05,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/29d78c745e8044a1a1ab4ff08f2ab17a is 50, key is test_row_0/C:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:05,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742448_1624 (size=12101) 2024-11-21T00:29:05,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/29d78c745e8044a1a1ab4ff08f2ab17a 2024-11-21T00:29:05,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/f6693a1b0c2746f5a019d96bb82fba81 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81 2024-11-21T00:29:05,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81, entries=200, sequenceid=133, filesize=38.7 K 2024-11-21T00:29:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/514578f2b5c340edae6b6b7cfc2df391 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/514578f2b5c340edae6b6b7cfc2df391 2024-11-21T00:29:05,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/514578f2b5c340edae6b6b7cfc2df391, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T00:29:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/29d78c745e8044a1a1ab4ff08f2ab17a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/29d78c745e8044a1a1ab4ff08f2ab17a 2024-11-21T00:29:05,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/29d78c745e8044a1a1ab4ff08f2ab17a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T00:29:05,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 619ceef852c1cdfc78ce20d02329c56f in 633ms, sequenceid=133, compaction requested=true 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:05,520 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:29:05,520 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:05,521 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:05,521 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:05,521 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:05,521 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:05,521 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:05,521 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:05,521 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/3c1a0e53e9bb40cc87eb70ada5052a3a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=99.4 K 2024-11-21T00:29:05,521 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b6255816e41843ba8156eab990c1e2fe, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5dfa59d1b1604f2e828d9debdac4d577, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/514578f2b5c340edae6b6b7cfc2df391] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=35.5 K 2024-11-21T00:29:05,521 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:05,521 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/3c1a0e53e9bb40cc87eb70ada5052a3a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81] 2024-11-21T00:29:05,522 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b6255816e41843ba8156eab990c1e2fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732148941376 2024-11-21T00:29:05,522 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1a0e53e9bb40cc87eb70ada5052a3a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732148941376 2024-11-21T00:29:05,522 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb580719d7c849a9abe4d981de5e68f7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148942489 2024-11-21T00:29:05,522 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfa59d1b1604f2e828d9debdac4d577, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148942489 2024-11-21T00:29:05,523 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 514578f2b5c340edae6b6b7cfc2df391, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732148943716 2024-11-21T00:29:05,523 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6693a1b0c2746f5a019d96bb82fba81, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732148943710 2024-11-21T00:29:05,528 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#527 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:05,529 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/113994a8b68f441ea15203f412f107ce is 50, key is test_row_0/B:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:05,548 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:05,550 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112145d01d7bde2a4cc09906ca8e3382676a_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:05,553 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112145d01d7bde2a4cc09906ca8e3382676a_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:05,553 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112145d01d7bde2a4cc09906ca8e3382676a_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:05,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:05,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T00:29:05,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742449_1625 (size=12409) 2024-11-21T00:29:05,595 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/113994a8b68f441ea15203f412f107ce as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113994a8b68f441ea15203f412f107ce 2024-11-21T00:29:05,600 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into 113994a8b68f441ea15203f412f107ce(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:05,600 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:05,600 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=13, startTime=1732148945520; duration=0sec 2024-11-21T00:29:05,600 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:05,600 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:05,600 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T00:29:05,601 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T00:29:05,601 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:05,601 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:05,601 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/80cc7901a43d47a38921858a3d2c2382, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/418fa2266148430fb520a55a4c6a70cc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/29d78c745e8044a1a1ab4ff08f2ab17a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=35.5 K 2024-11-21T00:29:05,602 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cc7901a43d47a38921858a3d2c2382, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732148941376 2024-11-21T00:29:05,602 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 418fa2266148430fb520a55a4c6a70cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732148942489 2024-11-21T00:29:05,602 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d78c745e8044a1a1ab4ff08f2ab17a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732148943716 2024-11-21T00:29:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742450_1626 (size=4469) 2024-11-21T00:29:05,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173cbcd246b8e4096a94500b24646f8e5_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148945565/Put/seqid=0 2024-11-21T00:29:05,609 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#528 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:05,611 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/aacb60c0ec5a48bc91818ede11f7daaa is 175, key is test_row_0/A:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:05,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149005608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,622 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:05,622 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/20ce9425bdba410baee78a2fba8afb58 is 50, key is test_row_0/C:col10/1732148943723/Put/seqid=0 2024-11-21T00:29:05,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149005618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149005621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149005664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742451_1627 (size=31363) 2024-11-21T00:29:05,696 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/aacb60c0ec5a48bc91818ede11f7daaa as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/aacb60c0ec5a48bc91818ede11f7daaa 2024-11-21T00:29:05,702 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into aacb60c0ec5a48bc91818ede11f7daaa(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:05,702 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:05,702 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=13, startTime=1732148945520; duration=0sec 2024-11-21T00:29:05,702 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:05,702 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:05,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149005724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742453_1629 (size=12409) 2024-11-21T00:29:05,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149005736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149005737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742452_1628 (size=14794) 2024-11-21T00:29:05,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149005932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149005952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149005952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,142 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/20ce9425bdba410baee78a2fba8afb58 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/20ce9425bdba410baee78a2fba8afb58 2024-11-21T00:29:06,150 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into 20ce9425bdba410baee78a2fba8afb58(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:06,150 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:06,150 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=13, startTime=1732148945520; duration=0sec 2024-11-21T00:29:06,150 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:06,150 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:06,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:06,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173cbcd246b8e4096a94500b24646f8e5_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173cbcd246b8e4096a94500b24646f8e5_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:06,169 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c22d969283d94800814d7f07a5788743, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:06,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c22d969283d94800814d7f07a5788743 is 175, key is test_row_0/A:col10/1732148945565/Put/seqid=0 2024-11-21T00:29:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149006182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742454_1630 (size=39749) 2024-11-21T00:29:06,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149006248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149006264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149006264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732149006612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,615 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c22d969283d94800814d7f07a5788743 2024-11-21T00:29:06,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/bd29fe10afcc497ab826c4dd9456c22c is 50, key is test_row_0/B:col10/1732148945565/Put/seqid=0 2024-11-21T00:29:06,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742455_1631 (size=12151) 2024-11-21T00:29:06,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/bd29fe10afcc497ab826c4dd9456c22c 2024-11-21T00:29:06,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/670f3a62b9ab4a3faf3de84e697553f0 is 50, key is test_row_0/C:col10/1732148945565/Put/seqid=0 2024-11-21T00:29:06,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149006763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149006772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149006772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:06,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742456_1632 (size=12151) 2024-11-21T00:29:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T00:29:06,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:06,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-21T00:29:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:29:06,845 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:06,846 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:06,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:29:07,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:29:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:07,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:07,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:29:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:07,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:29:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:07,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/670f3a62b9ab4a3faf3de84e697553f0 2024-11-21T00:29:07,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c22d969283d94800814d7f07a5788743 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743 2024-11-21T00:29:07,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743, entries=200, sequenceid=158, filesize=38.8 K 2024-11-21T00:29:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149007192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/bd29fe10afcc497ab826c4dd9456c22c as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/bd29fe10afcc497ab826c4dd9456c22c 2024-11-21T00:29:07,195 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/bd29fe10afcc497ab826c4dd9456c22c, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T00:29:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/670f3a62b9ab4a3faf3de84e697553f0 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/670f3a62b9ab4a3faf3de84e697553f0 2024-11-21T00:29:07,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/670f3a62b9ab4a3faf3de84e697553f0, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T00:29:07,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 619ceef852c1cdfc78ce20d02329c56f in 1674ms, sequenceid=158, compaction requested=false 2024-11-21T00:29:07,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:07,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T00:29:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:07,321 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:29:07,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112142463b41a1db40d4a35ab779c8ce09c5_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148945615/Put/seqid=0 2024-11-21T00:29:07,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742457_1633 (size=12304) 2024-11-21T00:29:07,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:07,381 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112142463b41a1db40d4a35ab779c8ce09c5_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112142463b41a1db40d4a35ab779c8ce09c5_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:07,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/43424983c00e42eea085eb8ea1303378, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:07,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/43424983c00e42eea085eb8ea1303378 is 175, key is test_row_0/A:col10/1732148945615/Put/seqid=0 2024-11-21T00:29:07,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742458_1634 (size=31105) 2024-11-21T00:29:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:29:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:07,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149007831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,841 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/43424983c00e42eea085eb8ea1303378 2024-11-21T00:29:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149007838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149007839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/00cc18ae5b94492db65a2ef4ac993051 is 50, key is test_row_0/B:col10/1732148945615/Put/seqid=0 2024-11-21T00:29:07,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742459_1635 (size=12151) 2024-11-21T00:29:07,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149007942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149007956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149007956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:29:08,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149008152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149008172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149008172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,301 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/00cc18ae5b94492db65a2ef4ac993051 2024-11-21T00:29:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/4fc6daad2cac471a8c88415a1b0e7657 is 50, key is test_row_0/C:col10/1732148945615/Put/seqid=0 2024-11-21T00:29:08,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742460_1636 (size=12151) 2024-11-21T00:29:08,381 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/4fc6daad2cac471a8c88415a1b0e7657 2024-11-21T00:29:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/43424983c00e42eea085eb8ea1303378 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378 2024-11-21T00:29:08,400 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378, entries=150, sequenceid=172, filesize=30.4 K 2024-11-21T00:29:08,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/00cc18ae5b94492db65a2ef4ac993051 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00cc18ae5b94492db65a2ef4ac993051 2024-11-21T00:29:08,404 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00cc18ae5b94492db65a2ef4ac993051, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T00:29:08,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/4fc6daad2cac471a8c88415a1b0e7657 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/4fc6daad2cac471a8c88415a1b0e7657 2024-11-21T00:29:08,411 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/4fc6daad2cac471a8c88415a1b0e7657, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T00:29:08,412 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 619ceef852c1cdfc78ce20d02329c56f in 1092ms, sequenceid=172, compaction requested=true 2024-11-21T00:29:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-21T00:29:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-21T00:29:08,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T00:29:08,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-21T00:29:08,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5760 sec 2024-11-21T00:29:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:08,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:08,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:29:08,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112196143cac0ef34ebc978270aaf2c79f35_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742461_1637 (size=14794) 2024-11-21T00:29:08,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:08,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112196143cac0ef34ebc978270aaf2c79f35_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112196143cac0ef34ebc978270aaf2c79f35_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:08,510 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/31de3cd9a6a847039beef55331e103fb, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:08,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/31de3cd9a6a847039beef55331e103fb is 175, key is test_row_0/A:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149008509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149008511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149008511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742462_1638 (size=39749) 2024-11-21T00:29:08,547 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/31de3cd9a6a847039beef55331e103fb 2024-11-21T00:29:08,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/7317796d282b4f15a54a6b2d59b54c86 is 50, key is test_row_0/B:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:08,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742463_1639 (size=12151) 2024-11-21T00:29:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149008624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149008625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149008625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149008843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149008843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149008843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T00:29:08,996 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:08,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-21T00:29:08,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:09,000 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:09,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:29:09,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/7317796d282b4f15a54a6b2d59b54c86 2024-11-21T00:29:09,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/901666994af842138e56bd68dea9d25f is 50, key is test_row_0/C:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:09,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742464_1640 (size=12151) 2024-11-21T00:29:09,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/901666994af842138e56bd68dea9d25f 2024-11-21T00:29:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:29:09,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/31de3cd9a6a847039beef55331e103fb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb 2024-11-21T00:29:09,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb, entries=200, sequenceid=198, filesize=38.8 K 2024-11-21T00:29:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/7317796d282b4f15a54a6b2d59b54c86 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/7317796d282b4f15a54a6b2d59b54c86 2024-11-21T00:29:09,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/7317796d282b4f15a54a6b2d59b54c86, entries=150, sequenceid=198, filesize=11.9 K 2024-11-21T00:29:09,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/901666994af842138e56bd68dea9d25f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/901666994af842138e56bd68dea9d25f 2024-11-21T00:29:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149009152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149009152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149009152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/901666994af842138e56bd68dea9d25f, entries=150, sequenceid=198, filesize=11.9 K 2024-11-21T00:29:09,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 619ceef852c1cdfc78ce20d02329c56f in 705ms, sequenceid=198, compaction requested=true 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:09,173 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:29:09,174 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:09,181 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:09,181 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:09,181 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,181 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113994a8b68f441ea15203f412f107ce, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/bd29fe10afcc497ab826c4dd9456c22c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00cc18ae5b94492db65a2ef4ac993051, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/7317796d282b4f15a54a6b2d59b54c86] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=47.7 K 2024-11-21T00:29:09,184 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:09,184 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:09,184 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,184 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/aacb60c0ec5a48bc91818ede11f7daaa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=138.6 K 2024-11-21T00:29:09,184 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,184 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/aacb60c0ec5a48bc91818ede11f7daaa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb] 2024-11-21T00:29:09,185 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 113994a8b68f441ea15203f412f107ce, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732148943716 2024-11-21T00:29:09,185 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting aacb60c0ec5a48bc91818ede11f7daaa, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732148943716 2024-11-21T00:29:09,188 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bd29fe10afcc497ab826c4dd9456c22c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732148944959 2024-11-21T00:29:09,188 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22d969283d94800814d7f07a5788743, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732148944959 2024-11-21T00:29:09,189 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 00cc18ae5b94492db65a2ef4ac993051, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732148945605 2024-11-21T00:29:09,189 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43424983c00e42eea085eb8ea1303378, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732148945605 2024-11-21T00:29:09,196 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31de3cd9a6a847039beef55331e103fb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732148947819 2024-11-21T00:29:09,199 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7317796d282b4f15a54a6b2d59b54c86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732148947836 2024-11-21T00:29:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:09,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:09,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T00:29:09,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121573e836aa7af44428db1b09430ff619d_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148948504/Put/seqid=0 2024-11-21T00:29:09,247 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#540 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:09,248 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:09,249 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/2f6ad0e4c2564f6c98e87283309c1188 is 50, key is test_row_0/B:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:09,254 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112162f36a12833e406d8538a6a4a0993b21_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:09,257 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112162f36a12833e406d8538a6a4a0993b21_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:09,257 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112162f36a12833e406d8538a6a4a0993b21_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:09,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742465_1641 (size=12304) 2024-11-21T00:29:09,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742466_1642 (size=12595) 2024-11-21T00:29:09,300 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121573e836aa7af44428db1b09430ff619d_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121573e836aa7af44428db1b09430ff619d_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:09,307 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c71399f2135a481e98d329279ea76fad, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:09,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c71399f2135a481e98d329279ea76fad is 175, key is test_row_0/A:col10/1732148948504/Put/seqid=0 2024-11-21T00:29:09,320 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/2f6ad0e4c2564f6c98e87283309c1188 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/2f6ad0e4c2564f6c98e87283309c1188 2024-11-21T00:29:09,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:09,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:09,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:29:09,330 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into 2f6ad0e4c2564f6c98e87283309c1188(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:09,330 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:09,330 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=12, startTime=1732148949173; duration=0sec 2024-11-21T00:29:09,330 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:09,331 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:09,331 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742467_1643 (size=4469) 2024-11-21T00:29:09,332 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:09,332 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:09,332 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,332 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/20ce9425bdba410baee78a2fba8afb58, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/670f3a62b9ab4a3faf3de84e697553f0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/4fc6daad2cac471a8c88415a1b0e7657, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/901666994af842138e56bd68dea9d25f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=47.7 K 2024-11-21T00:29:09,337 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ce9425bdba410baee78a2fba8afb58, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732148943716 2024-11-21T00:29:09,338 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 670f3a62b9ab4a3faf3de84e697553f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732148944959 2024-11-21T00:29:09,338 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fc6daad2cac471a8c88415a1b0e7657, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732148945605 2024-11-21T00:29:09,339 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 901666994af842138e56bd68dea9d25f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732148947836 2024-11-21T00:29:09,372 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#541 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:09,373 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/0f927685864449f5a64e360a4245382f is 175, key is test_row_0/A:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:09,382 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#542 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:09,383 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/38643824c4a343699e0174235d0639a5 is 50, key is test_row_0/C:col10/1732148947837/Put/seqid=0 2024-11-21T00:29:09,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742468_1644 (size=31105) 2024-11-21T00:29:09,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742469_1645 (size=31549) 2024-11-21T00:29:09,431 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/0f927685864449f5a64e360a4245382f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/0f927685864449f5a64e360a4245382f 2024-11-21T00:29:09,436 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into 0f927685864449f5a64e360a4245382f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:09,436 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=12, startTime=1732148949173; duration=0sec 2024-11-21T00:29:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:09,436 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:09,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742470_1646 (size=12595) 2024-11-21T00:29:09,445 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/38643824c4a343699e0174235d0639a5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/38643824c4a343699e0174235d0639a5 2024-11-21T00:29:09,451 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into 38643824c4a343699e0174235d0639a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:09,451 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:09,451 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=12, startTime=1732148949173; duration=0sec 2024-11-21T00:29:09,451 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:09,451 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:09,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149009474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149009585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:29:09,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149009665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149009679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149009679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c71399f2135a481e98d329279ea76fad 2024-11-21T00:29:09,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149009792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:09,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/113c19d5e4ef4583af3ec85b9e114414 is 50, key is test_row_0/B:col10/1732148948504/Put/seqid=0 2024-11-21T00:29:09,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742471_1647 (size=12151) 2024-11-21T00:29:09,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:09,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149010102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:10,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:10,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:29:10,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/113c19d5e4ef4583af3ec85b9e114414 2024-11-21T00:29:10,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/acddc336641f4357a4b8d5fc4d781ac6 is 50, key is test_row_0/C:col10/1732148948504/Put/seqid=0 2024-11-21T00:29:10,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742472_1648 (size=12151) 2024-11-21T00:29:10,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149010612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149010682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:10,687 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:10,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/acddc336641f4357a4b8d5fc4d781ac6 2024-11-21T00:29:10,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/c71399f2135a481e98d329279ea76fad as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad 2024-11-21T00:29:10,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad, entries=150, sequenceid=211, filesize=30.4 K 2024-11-21T00:29:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149010693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:10,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/113c19d5e4ef4583af3ec85b9e114414 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113c19d5e4ef4583af3ec85b9e114414 2024-11-21T00:29:10,702 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:10,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113c19d5e4ef4583af3ec85b9e114414, entries=150, sequenceid=211, filesize=11.9 K 2024-11-21T00:29:10,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149010693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:10,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/acddc336641f4357a4b8d5fc4d781ac6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/acddc336641f4357a4b8d5fc4d781ac6 2024-11-21T00:29:10,709 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_2' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:10,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/acddc336641f4357a4b8d5fc4d781ac6, entries=150, sequenceid=211, filesize=11.9 K 2024-11-21T00:29:10,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 619ceef852c1cdfc78ce20d02329c56f in 1511ms, sequenceid=211, compaction requested=false 2024-11-21T00:29:10,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:10,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T00:29:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:10,738 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T00:29:10,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121177086f43d674c52a8cba5855bf7ee02_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148949416/Put/seqid=0 2024-11-21T00:29:10,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742473_1649 (size=12304) 2024-11-21T00:29:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,820 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121177086f43d674c52a8cba5855bf7ee02_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121177086f43d674c52a8cba5855bf7ee02_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/69818aeae0024eafb18b9c14e26b600a, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:10,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/69818aeae0024eafb18b9c14e26b600a is 175, key is test_row_0/A:col10/1732148949416/Put/seqid=0 2024-11-21T00:29:10,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742474_1650 (size=31105) 2024-11-21T00:29:10,869 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/69818aeae0024eafb18b9c14e26b600a 2024-11-21T00:29:10,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/6f153ccc69b04ddcb1633194828cd8d8 is 50, key is test_row_0/B:col10/1732148949416/Put/seqid=0 2024-11-21T00:29:10,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742475_1651 (size=12151) 2024-11-21T00:29:10,934 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/6f153ccc69b04ddcb1633194828cd8d8 2024-11-21T00:29:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/0fffb728b60f420dafd8d2fccae469a6 is 50, key is test_row_0/C:col10/1732148949416/Put/seqid=0 2024-11-21T00:29:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742476_1652 (size=12151) 2024-11-21T00:29:10,970 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/0fffb728b60f420dafd8d2fccae469a6 2024-11-21T00:29:10,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/69818aeae0024eafb18b9c14e26b600a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a 2024-11-21T00:29:10,981 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a, entries=150, sequenceid=237, filesize=30.4 K 2024-11-21T00:29:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/6f153ccc69b04ddcb1633194828cd8d8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f153ccc69b04ddcb1633194828cd8d8 2024-11-21T00:29:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,994 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f153ccc69b04ddcb1633194828cd8d8, entries=150, sequenceid=237, filesize=11.9 K 2024-11-21T00:29:10,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/0fffb728b60f420dafd8d2fccae469a6 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/0fffb728b60f420dafd8d2fccae469a6 2024-11-21T00:29:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,001 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/0fffb728b60f420dafd8d2fccae469a6, entries=150, sequenceid=237, filesize=11.9 K 2024-11-21T00:29:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,003 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 619ceef852c1cdfc78ce20d02329c56f in 265ms, sequenceid=237, compaction requested=true 2024-11-21T00:29:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-21T00:29:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-21T00:29:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-21T00:29:11,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-21T00:29:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.0130 sec 2024-11-21T00:29:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T00:29:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,153 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T00:29:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,157 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:11,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:29:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:29:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T00:29:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T00:29:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T00:29:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T00:29:11,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-21T00:29:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 160 msec 2024-11-21T00:29:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T00:29:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,473 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T00:29:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:29:11,475 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:11,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:29:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T00:29:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T00:29:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T00:29:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T00:29:11,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 160 msec 2024-11-21T00:29:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 169 msec 2024-11-21T00:29:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:11,722 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:11,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:29:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:11,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c27eee7dbcd84c5eb88dfb9d21b98d8c_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:11,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742477_1653 (size=14794) 2024-11-21T00:29:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T00:29:11,792 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:11,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-21T00:29:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:29:11,796 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:11,796 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:11,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:11,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149011892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:29:11,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:11,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149012008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:29:12,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:12,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c27eee7dbcd84c5eb88dfb9d21b98d8c_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c27eee7dbcd84c5eb88dfb9d21b98d8c_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:12,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/609fe72a3b7547b8a3dac291f31e5912, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/609fe72a3b7547b8a3dac291f31e5912 is 175, key is test_row_0/A:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:12,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742478_1654 (size=39749) 2024-11-21T00:29:12,222 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/609fe72a3b7547b8a3dac291f31e5912 2024-11-21T00:29:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149012223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/231916760bec4027ba824785d8539eac is 50, key is test_row_0/B:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:12,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742479_1655 (size=12151) 2024-11-21T00:29:12,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:29:12,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149012531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/231916760bec4027ba824785d8539eac 2024-11-21T00:29:12,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/9b81672496314d2d91e935cb3c35fe1f is 50, key is test_row_0/C:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742480_1656 (size=12151) 2024-11-21T00:29:12,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/9b81672496314d2d91e935cb3c35fe1f 2024-11-21T00:29:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/609fe72a3b7547b8a3dac291f31e5912 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912 2024-11-21T00:29:12,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912, entries=200, sequenceid=250, filesize=38.8 K 2024-11-21T00:29:12,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/231916760bec4027ba824785d8539eac as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/231916760bec4027ba824785d8539eac 2024-11-21T00:29:12,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/231916760bec4027ba824785d8539eac, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T00:29:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/9b81672496314d2d91e935cb3c35fe1f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/9b81672496314d2d91e935cb3c35fe1f 2024-11-21T00:29:12,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/9b81672496314d2d91e935cb3c35fe1f, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T00:29:12,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 619ceef852c1cdfc78ce20d02329c56f in 956ms, sequenceid=250, compaction requested=true 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:12,678 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:12,678 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:12,680 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133508 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:12,680 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:12,680 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:12,680 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:12,680 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,680 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,680 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/2f6ad0e4c2564f6c98e87283309c1188, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113c19d5e4ef4583af3ec85b9e114414, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f153ccc69b04ddcb1633194828cd8d8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/231916760bec4027ba824785d8539eac] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=47.9 K 2024-11-21T00:29:12,680 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/0f927685864449f5a64e360a4245382f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=130.4 K 2024-11-21T00:29:12,680 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,680 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/0f927685864449f5a64e360a4245382f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912] 2024-11-21T00:29:12,681 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6ad0e4c2564f6c98e87283309c1188, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732148947836 2024-11-21T00:29:12,681 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f927685864449f5a64e360a4245382f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732148947836 2024-11-21T00:29:12,681 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 113c19d5e4ef4583af3ec85b9e114414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732148948504 2024-11-21T00:29:12,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f153ccc69b04ddcb1633194828cd8d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732148949416 2024-11-21T00:29:12,682 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c71399f2135a481e98d329279ea76fad, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732148948504 2024-11-21T00:29:12,682 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 231916760bec4027ba824785d8539eac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148951693 2024-11-21T00:29:12,682 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69818aeae0024eafb18b9c14e26b600a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732148949416 2024-11-21T00:29:12,682 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 609fe72a3b7547b8a3dac291f31e5912, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148951693 2024-11-21T00:29:12,688 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#551 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:12,689 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/4bb14e0f7f1c4507829900d6ff90290e is 50, key is test_row_0/B:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:12,689 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,701 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ac5d36500bcb40ec9c77c4059c17165e_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,703 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ac5d36500bcb40ec9c77c4059c17165e_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,703 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ac5d36500bcb40ec9c77c4059c17165e_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742481_1657 (size=12731) 2024-11-21T00:29:12,712 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/4bb14e0f7f1c4507829900d6ff90290e as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/4bb14e0f7f1c4507829900d6ff90290e 2024-11-21T00:29:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:12,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T00:29:12,717 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into 4bb14e0f7f1c4507829900d6ff90290e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:12,717 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:12,717 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=12, startTime=1732148952678; duration=0sec 2024-11-21T00:29:12,718 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:12,718 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:12,718 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:12,718 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:12,719 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:12,719 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,719 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/38643824c4a343699e0174235d0639a5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/acddc336641f4357a4b8d5fc4d781ac6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/0fffb728b60f420dafd8d2fccae469a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/9b81672496314d2d91e935cb3c35fe1f] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=47.9 K 2024-11-21T00:29:12,719 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 38643824c4a343699e0174235d0639a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732148947836 2024-11-21T00:29:12,719 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting acddc336641f4357a4b8d5fc4d781ac6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732148948504 2024-11-21T00:29:12,719 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fffb728b60f420dafd8d2fccae469a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732148949416 2024-11-21T00:29:12,720 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b81672496314d2d91e935cb3c35fe1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148951693 2024-11-21T00:29:12,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:12,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:12,731 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#553 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:12,732 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bdd1c5d298874455b8d31c088c1de946 is 50, key is test_row_0/C:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149012733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149012735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a2de241320e24d9d96a05bcb4a504606_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148951874/Put/seqid=0 2024-11-21T00:29:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742483_1659 (size=12731) 2024-11-21T00:29:12,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742482_1658 (size=4469) 2024-11-21T00:29:12,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149012739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,742 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#552 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:12,742 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/df51e1fd00c94032abb0904cc4cea60b is 175, key is test_row_0/A:col10/1732148951716/Put/seqid=0 2024-11-21T00:29:12,746 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bdd1c5d298874455b8d31c088c1de946 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bdd1c5d298874455b8d31c088c1de946 2024-11-21T00:29:12,750 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into bdd1c5d298874455b8d31c088c1de946(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:12,750 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:12,750 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=12, startTime=1732148952678; duration=0sec 2024-11-21T00:29:12,750 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:12,750 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:12,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742484_1660 (size=12454) 2024-11-21T00:29:12,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:12,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a2de241320e24d9d96a05bcb4a504606_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2de241320e24d9d96a05bcb4a504606_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:12,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/4d265ff98893443ab82989fd393a3d9f, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/4d265ff98893443ab82989fd393a3d9f is 175, key is test_row_0/A:col10/1732148951874/Put/seqid=0 2024-11-21T00:29:12,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742485_1661 (size=31685) 2024-11-21T00:29:12,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742486_1662 (size=31255) 2024-11-21T00:29:12,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/4d265ff98893443ab82989fd393a3d9f 2024-11-21T00:29:12,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ce2db34abeb342719e93e228c3cd333f is 50, key is test_row_0/B:col10/1732148951874/Put/seqid=0 2024-11-21T00:29:12,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742487_1663 (size=12301) 2024-11-21T00:29:12,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ce2db34abeb342719e93e228c3cd333f 2024-11-21T00:29:12,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bc2c17bb0fd04c6286a298fc102aac96 is 50, key is test_row_0/C:col10/1732148951874/Put/seqid=0 2024-11-21T00:29:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149012842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149012842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742488_1664 (size=12301) 2024-11-21T00:29:12,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bc2c17bb0fd04c6286a298fc102aac96 2024-11-21T00:29:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149012851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:12,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/4d265ff98893443ab82989fd393a3d9f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f 2024-11-21T00:29:12,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f, entries=150, sequenceid=276, filesize=30.5 K 2024-11-21T00:29:12,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ce2db34abeb342719e93e228c3cd333f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ce2db34abeb342719e93e228c3cd333f 2024-11-21T00:29:12,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ce2db34abeb342719e93e228c3cd333f, entries=150, sequenceid=276, filesize=12.0 K 2024-11-21T00:29:12,868 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T00:29:12,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/bc2c17bb0fd04c6286a298fc102aac96 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bc2c17bb0fd04c6286a298fc102aac96 2024-11-21T00:29:12,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bc2c17bb0fd04c6286a298fc102aac96, entries=150, sequenceid=276, filesize=12.0 K 2024-11-21T00:29:12,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T00:29:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 619ceef852c1cdfc78ce20d02329c56f in 160ms, sequenceid=276, compaction requested=false 2024-11-21T00:29:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:12,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:12,874 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T00:29:12,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211aed6dfc75a148d1a8fb02c4b0850619_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148952725/Put/seqid=0 2024-11-21T00:29:12,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742489_1665 (size=12454) 2024-11-21T00:29:12,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:12,927 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211aed6dfc75a148d1a8fb02c4b0850619_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211aed6dfc75a148d1a8fb02c4b0850619_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:12,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d25a52b5d36a48199fa0ce3263886ae9, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:12,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d25a52b5d36a48199fa0ce3263886ae9 is 175, key is test_row_0/A:col10/1732148952725/Put/seqid=0 2024-11-21T00:29:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:29:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742490_1666 (size=31255) 2024-11-21T00:29:13,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149013092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149013094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149013099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149013100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,179 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/df51e1fd00c94032abb0904cc4cea60b as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/df51e1fd00c94032abb0904cc4cea60b 2024-11-21T00:29:13,185 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into df51e1fd00c94032abb0904cc4cea60b(size=30.9 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:13,185 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:13,185 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=12, startTime=1732148952678; duration=0sec 2024-11-21T00:29:13,185 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:13,185 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:13,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149013202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149013212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149013212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149013212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,346 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d25a52b5d36a48199fa0ce3263886ae9 2024-11-21T00:29:13,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/6f97fccf35e14dd3856f0a6dbbb5d637 is 50, key is test_row_0/B:col10/1732148952725/Put/seqid=0 2024-11-21T00:29:13,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742491_1667 (size=12301) 2024-11-21T00:29:13,368 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/6f97fccf35e14dd3856f0a6dbbb5d637 2024-11-21T00:29:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/c0a9cab015b848aeb3cd8c6829716d6a is 50, key is test_row_0/C:col10/1732148952725/Put/seqid=0 2024-11-21T00:29:13,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742492_1668 (size=12301) 2024-11-21T00:29:13,394 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/c0a9cab015b848aeb3cd8c6829716d6a 2024-11-21T00:29:13,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d25a52b5d36a48199fa0ce3263886ae9 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9 2024-11-21T00:29:13,401 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9, entries=150, sequenceid=288, filesize=30.5 K 2024-11-21T00:29:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/6f97fccf35e14dd3856f0a6dbbb5d637 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f97fccf35e14dd3856f0a6dbbb5d637 2024-11-21T00:29:13,405 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f97fccf35e14dd3856f0a6dbbb5d637, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T00:29:13,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/c0a9cab015b848aeb3cd8c6829716d6a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c0a9cab015b848aeb3cd8c6829716d6a 2024-11-21T00:29:13,411 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c0a9cab015b848aeb3cd8c6829716d6a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T00:29:13,411 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 619ceef852c1cdfc78ce20d02329c56f in 537ms, sequenceid=288, compaction requested=true 2024-11-21T00:29:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-21T00:29:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-21T00:29:13,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T00:29:13,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-21T00:29:13,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.6270 sec 2024-11-21T00:29:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:13,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:13,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:29:13,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121274cbb91a9304c618c14bac19bcee0bd_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149013445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149013445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149013446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149013451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742493_1669 (size=14994) 2024-11-21T00:29:13,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:13,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121274cbb91a9304c618c14bac19bcee0bd_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121274cbb91a9304c618c14bac19bcee0bd_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:13,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d15f17e5d865497cba815c37be6ab36a, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:13,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d15f17e5d865497cba815c37be6ab36a is 175, key is test_row_0/A:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:13,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742494_1670 (size=39949) 2024-11-21T00:29:13,507 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d15f17e5d865497cba815c37be6ab36a 2024-11-21T00:29:13,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b47d8d2df4294c608fb995e05b3f44f2 is 50, key is test_row_0/B:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:13,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742495_1671 (size=12301) 2024-11-21T00:29:13,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b47d8d2df4294c608fb995e05b3f44f2 2024-11-21T00:29:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149013562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149013562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149013564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149013564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/2f93aefed5ba4a4ca74a6432e62b3eb8 is 50, key is test_row_0/C:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742496_1672 (size=12301) 2024-11-21T00:29:13,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149013772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149013772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149013772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149013773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T00:29:13,943 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:13,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-21T00:29:13,945 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T00:29:13,946 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:13,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:13,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/2f93aefed5ba4a4ca74a6432e62b3eb8 2024-11-21T00:29:13,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/d15f17e5d865497cba815c37be6ab36a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a 2024-11-21T00:29:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a, entries=200, sequenceid=316, filesize=39.0 K 2024-11-21T00:29:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b47d8d2df4294c608fb995e05b3f44f2 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b47d8d2df4294c608fb995e05b3f44f2 2024-11-21T00:29:13,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b47d8d2df4294c608fb995e05b3f44f2, entries=150, sequenceid=316, filesize=12.0 K 2024-11-21T00:29:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/2f93aefed5ba4a4ca74a6432e62b3eb8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f93aefed5ba4a4ca74a6432e62b3eb8 2024-11-21T00:29:13,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f93aefed5ba4a4ca74a6432e62b3eb8, entries=150, sequenceid=316, filesize=12.0 K 2024-11-21T00:29:13,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 619ceef852c1cdfc78ce20d02329c56f in 566ms, sequenceid=316, compaction requested=true 2024-11-21T00:29:13,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:13,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:13,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:13,998 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:13,998 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:13,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:13,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:13,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:13,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:13,999 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:13,999 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:13,999 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:13,999 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/4bb14e0f7f1c4507829900d6ff90290e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ce2db34abeb342719e93e228c3cd333f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f97fccf35e14dd3856f0a6dbbb5d637, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b47d8d2df4294c608fb995e05b3f44f2] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=48.5 K 2024-11-21T00:29:13,999 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:13,999 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:13,999 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:13,999 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/df51e1fd00c94032abb0904cc4cea60b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=131 K 2024-11-21T00:29:13,999 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,000 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/df51e1fd00c94032abb0904cc4cea60b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a] 2024-11-21T00:29:14,002 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting df51e1fd00c94032abb0904cc4cea60b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148951693 2024-11-21T00:29:14,002 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb14e0f7f1c4507829900d6ff90290e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148951693 2024-11-21T00:29:14,003 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2db34abeb342719e93e228c3cd333f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732148951874 2024-11-21T00:29:14,003 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d265ff98893443ab82989fd393a3d9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732148951874 2024-11-21T00:29:14,003 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f97fccf35e14dd3856f0a6dbbb5d637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148952725 2024-11-21T00:29:14,003 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25a52b5d36a48199fa0ce3263886ae9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148952725 2024-11-21T00:29:14,003 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b47d8d2df4294c608fb995e05b3f44f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732148953075 2024-11-21T00:29:14,004 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15f17e5d865497cba815c37be6ab36a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732148953075 2024-11-21T00:29:14,037 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#563 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:14,038 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/9688189f16bb49aab28657257749b9b8 is 50, key is test_row_0/B:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:14,045 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T00:29:14,053 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112186309b2cee6a450886c9afeb37a1b540_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:14,056 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112186309b2cee6a450886c9afeb37a1b540_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:14,056 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112186309b2cee6a450886c9afeb37a1b540_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:14,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742497_1673 (size=13017) 2024-11-21T00:29:14,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:14,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:29:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:14,101 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/9688189f16bb49aab28657257749b9b8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9688189f16bb49aab28657257749b9b8 2024-11-21T00:29:14,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182458c2946e543b2a46352d0ed488081_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148954089/Put/seqid=0 2024-11-21T00:29:14,128 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into 9688189f16bb49aab28657257749b9b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:14,128 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:14,128 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=12, startTime=1732148953998; duration=0sec 2024-11-21T00:29:14,128 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:14,128 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:14,128 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:14,130 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:14,131 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:14,131 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,131 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bdd1c5d298874455b8d31c088c1de946, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bc2c17bb0fd04c6286a298fc102aac96, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c0a9cab015b848aeb3cd8c6829716d6a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f93aefed5ba4a4ca74a6432e62b3eb8] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=48.5 K 2024-11-21T00:29:14,131 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd1c5d298874455b8d31c088c1de946, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732148951693 2024-11-21T00:29:14,133 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting bc2c17bb0fd04c6286a298fc102aac96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732148951874 2024-11-21T00:29:14,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742498_1674 (size=4469) 2024-11-21T00:29:14,134 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting c0a9cab015b848aeb3cd8c6829716d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732148952725 2024-11-21T00:29:14,135 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f93aefed5ba4a4ca74a6432e62b3eb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732148953075 2024-11-21T00:29:14,158 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#566 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:14,159 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7 is 50, key is test_row_0/C:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:14,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742499_1675 (size=12454) 2024-11-21T00:29:14,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742500_1676 (size=13017) 2024-11-21T00:29:14,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182458c2946e543b2a46352d0ed488081_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182458c2946e543b2a46352d0ed488081_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:14,197 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b2255607e1b7481ea157b46b4e5ed6d4, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:14,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b2255607e1b7481ea157b46b4e5ed6d4 is 175, key is test_row_0/A:col10/1732148954089/Put/seqid=0 2024-11-21T00:29:14,208 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7 2024-11-21T00:29:14,220 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into e403b7d1c5d54b47b7bd4fc22bb6f9a7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:14,220 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:14,220 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=12, startTime=1732148953998; duration=0sec 2024-11-21T00:29:14,220 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:14,220 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149014219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149014219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149014222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149014229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742501_1677 (size=31255) 2024-11-21T00:29:14,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b2255607e1b7481ea157b46b4e5ed6d4 2024-11-21T00:29:14,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T00:29:14,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/3d129c8b46d34b5590698e55c9da8024 is 50, key is test_row_0/B:col10/1732148954089/Put/seqid=0 2024-11-21T00:29:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742502_1678 (size=12301) 2024-11-21T00:29:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149014332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149014332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149014342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149014342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,536 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#564 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:14,536 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/f0d73bdf244748e0ae2ba89c2d280944 is 175, key is test_row_0/A:col10/1732148953075/Put/seqid=0 2024-11-21T00:29:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149014542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149014542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149014552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149014556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T00:29:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742503_1679 (size=31971) 2024-11-21T00:29:14,605 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/f0d73bdf244748e0ae2ba89c2d280944 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f0d73bdf244748e0ae2ba89c2d280944 2024-11-21T00:29:14,610 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into f0d73bdf244748e0ae2ba89c2d280944(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:14,610 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:14,610 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=12, startTime=1732148953998; duration=0sec 2024-11-21T00:29:14,611 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:14,611 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:14,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/3d129c8b46d34b5590698e55c9da8024 2024-11-21T00:29:14,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ec10419847814b28b4e82d583dcd0736 is 50, key is test_row_0/C:col10/1732148954089/Put/seqid=0 2024-11-21T00:29:14,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742504_1680 (size=12301) 2024-11-21T00:29:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149014851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149014852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149014862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149014872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:14,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:14,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T00:29:15,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ec10419847814b28b4e82d583dcd0736 2024-11-21T00:29:15,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/b2255607e1b7481ea157b46b4e5ed6d4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4 2024-11-21T00:29:15,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4, entries=150, sequenceid=328, filesize=30.5 K 2024-11-21T00:29:15,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/3d129c8b46d34b5590698e55c9da8024 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/3d129c8b46d34b5590698e55c9da8024 2024-11-21T00:29:15,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:15,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/3d129c8b46d34b5590698e55c9da8024, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T00:29:15,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ec10419847814b28b4e82d583dcd0736 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ec10419847814b28b4e82d583dcd0736 2024-11-21T00:29:15,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ec10419847814b28b4e82d583dcd0736, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T00:29:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 619ceef852c1cdfc78ce20d02329c56f in 1116ms, sequenceid=328, compaction requested=false 2024-11-21T00:29:15,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:15,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T00:29:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,342 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T00:29:15,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e4329e7130b946dfbfec33973de999aa_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148954227/Put/seqid=0 2024-11-21T00:29:15,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742505_1681 (size=12454) 2024-11-21T00:29:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:15,391 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e4329e7130b946dfbfec33973de999aa_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4329e7130b946dfbfec33973de999aa_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1dd0a329d19442b1b28cf6b1311e45c5, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:15,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1dd0a329d19442b1b28cf6b1311e45c5 is 175, key is test_row_0/A:col10/1732148954227/Put/seqid=0 2024-11-21T00:29:15,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149015387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149015387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149015389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149015391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742506_1682 (size=31255) 2024-11-21T00:29:15,411 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1dd0a329d19442b1b28cf6b1311e45c5 2024-11-21T00:29:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/e8090b902a3645da9205d022a3475159 is 50, key is test_row_0/B:col10/1732148954227/Put/seqid=0 2024-11-21T00:29:15,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742507_1683 (size=12301) 2024-11-21T00:29:15,474 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/e8090b902a3645da9205d022a3475159 2024-11-21T00:29:15,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/657caa56d70b41f1b174c2eb954acb9d is 50, key is test_row_0/C:col10/1732148954227/Put/seqid=0 2024-11-21T00:29:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742508_1684 (size=12301) 2024-11-21T00:29:15,502 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/657caa56d70b41f1b174c2eb954acb9d 2024-11-21T00:29:15,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149015502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149015502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149015502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/1dd0a329d19442b1b28cf6b1311e45c5 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5 2024-11-21T00:29:15,511 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5, entries=150, sequenceid=355, filesize=30.5 K 2024-11-21T00:29:15,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/e8090b902a3645da9205d022a3475159 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/e8090b902a3645da9205d022a3475159 2024-11-21T00:29:15,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149015512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,518 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/e8090b902a3645da9205d022a3475159, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T00:29:15,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/657caa56d70b41f1b174c2eb954acb9d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/657caa56d70b41f1b174c2eb954acb9d 2024-11-21T00:29:15,540 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/657caa56d70b41f1b174c2eb954acb9d, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T00:29:15,542 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 619ceef852c1cdfc78ce20d02329c56f in 200ms, sequenceid=355, compaction requested=true 2024-11-21T00:29:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-21T00:29:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-21T00:29:15,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T00:29:15,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-21T00:29:15,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.6050 sec 2024-11-21T00:29:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:15,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:15,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T00:29:15,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a1e82b239b4346af9ab727d742c5923a_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:15,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742509_1685 (size=17534) 2024-11-21T00:29:15,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:15,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a1e82b239b4346af9ab727d742c5923a_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a1e82b239b4346af9ab727d742c5923a_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:15,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/13c98726acdc4cce98789bae3428d346, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:15,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/13c98726acdc4cce98789bae3428d346 is 175, key is test_row_0/A:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:15,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149015772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149015772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149015774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149015775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742510_1686 (size=48639) 2024-11-21T00:29:15,816 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/13c98726acdc4cce98789bae3428d346 2024-11-21T00:29:15,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ebd2761412de4f43be0e67f0f05189de is 50, key is test_row_0/B:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742511_1687 (size=12301) 2024-11-21T00:29:15,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ebd2761412de4f43be0e67f0f05189de 2024-11-21T00:29:15,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/1424e634de344bbf86dd821f1656b4fb is 50, key is test_row_0/C:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:15,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149015882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149015892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149015892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149015893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:15,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742512_1688 (size=12301) 2024-11-21T00:29:15,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/1424e634de344bbf86dd821f1656b4fb 2024-11-21T00:29:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/13c98726acdc4cce98789bae3428d346 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346 2024-11-21T00:29:15,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346, entries=250, sequenceid=369, filesize=47.5 K 2024-11-21T00:29:15,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/ebd2761412de4f43be0e67f0f05189de as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ebd2761412de4f43be0e67f0f05189de 2024-11-21T00:29:15,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ebd2761412de4f43be0e67f0f05189de, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T00:29:15,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/1424e634de344bbf86dd821f1656b4fb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/1424e634de344bbf86dd821f1656b4fb 2024-11-21T00:29:15,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/1424e634de344bbf86dd821f1656b4fb, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T00:29:15,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 619ceef852c1cdfc78ce20d02329c56f in 214ms, sequenceid=369, compaction requested=true 2024-11-21T00:29:15,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:15,931 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:15,934 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:15,934 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:15,934 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,934 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f0d73bdf244748e0ae2ba89c2d280944, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=139.8 K 2024-11-21T00:29:15,934 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,934 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f0d73bdf244748e0ae2ba89c2d280944, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346] 2024-11-21T00:29:15,936 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d73bdf244748e0ae2ba89c2d280944, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732148953075 2024-11-21T00:29:15,936 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2255607e1b7481ea157b46b4e5ed6d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732148953442 2024-11-21T00:29:15,936 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dd0a329d19442b1b28cf6b1311e45c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732148954148 2024-11-21T00:29:15,937 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13c98726acdc4cce98789bae3428d346, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148955376 2024-11-21T00:29:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T00:29:15,940 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:15,943 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:15,943 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:15,943 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:15,943 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9688189f16bb49aab28657257749b9b8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/3d129c8b46d34b5590698e55c9da8024, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/e8090b902a3645da9205d022a3475159, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ebd2761412de4f43be0e67f0f05189de] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=48.8 K 2024-11-21T00:29:15,943 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9688189f16bb49aab28657257749b9b8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732148953075 2024-11-21T00:29:15,943 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d129c8b46d34b5590698e55c9da8024, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732148953442 2024-11-21T00:29:15,944 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e8090b902a3645da9205d022a3475159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732148954148 2024-11-21T00:29:15,944 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd2761412de4f43be0e67f0f05189de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148955385 2024-11-21T00:29:15,961 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:15,967 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#576 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:15,968 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/00d575e1654c4e87802ecbf433d3ce43 is 50, key is test_row_0/B:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:15,977 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f720382c08e244b7b4bbf283df54c454_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:15,984 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f720382c08e244b7b4bbf283df54c454_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:15,984 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f720382c08e244b7b4bbf283df54c454_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:16,008 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T00:29:16,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742513_1689 (size=13153) 2024-11-21T00:29:16,031 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/00d575e1654c4e87802ecbf433d3ce43 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00d575e1654c4e87802ecbf433d3ce43 2024-11-21T00:29:16,040 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into 00d575e1654c4e87802ecbf433d3ce43(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:16,040 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:16,040 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=12, startTime=1732148955940; duration=0sec 2024-11-21T00:29:16,040 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:16,040 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:16,040 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:16,042 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:16,042 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:16,042 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,042 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ec10419847814b28b4e82d583dcd0736, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/657caa56d70b41f1b174c2eb954acb9d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/1424e634de344bbf86dd821f1656b4fb] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=48.8 K 2024-11-21T00:29:16,042 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting e403b7d1c5d54b47b7bd4fc22bb6f9a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732148953075 2024-11-21T00:29:16,043 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting ec10419847814b28b4e82d583dcd0736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732148953442 2024-11-21T00:29:16,043 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 657caa56d70b41f1b174c2eb954acb9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732148954148 2024-11-21T00:29:16,044 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1424e634de344bbf86dd821f1656b4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148955385 2024-11-21T00:29:16,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742514_1690 (size=4469) 2024-11-21T00:29:16,051 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#575 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:16,052 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/85d9f5c98e5849c3a9463a435556e953 is 175, key is test_row_0/A:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T00:29:16,102 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:16,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-21T00:29:16,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:16,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742515_1691 (size=32107) 2024-11-21T00:29:16,107 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:16,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:29:16,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:16,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T00:29:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:16,114 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/85d9f5c98e5849c3a9463a435556e953 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/85d9f5c98e5849c3a9463a435556e953 2024-11-21T00:29:16,120 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into 85d9f5c98e5849c3a9463a435556e953(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:16,120 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:16,120 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=12, startTime=1732148955931; duration=0sec 2024-11-21T00:29:16,121 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:16,121 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:16,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213170fd0d97b1431499c43353a5da9027_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148956108/Put/seqid=0 2024-11-21T00:29:16,128 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#577 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:16,129 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/6e46b16cb0474d28a89ede0a1fe031b8 is 50, key is test_row_0/C:col10/1732148955715/Put/seqid=0 2024-11-21T00:29:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149016131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149016132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149016133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149016134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742516_1692 (size=12454) 2024-11-21T00:29:16,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742517_1693 (size=13153) 2024-11-21T00:29:16,178 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/6e46b16cb0474d28a89ede0a1fe031b8 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/6e46b16cb0474d28a89ede0a1fe031b8 2024-11-21T00:29:16,183 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into 6e46b16cb0474d28a89ede0a1fe031b8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:16,183 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:16,183 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=12, startTime=1732148955940; duration=0sec 2024-11-21T00:29:16,183 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:16,183 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:29:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149016242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149016243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149016245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149016252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T00:29:16,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T00:29:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:29:16,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149016452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149016454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149016455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149016468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:16,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213170fd0d97b1431499c43353a5da9027_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213170fd0d97b1431499c43353a5da9027_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:16,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6df3a56296bd4739beba3189a2879477, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:16,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6df3a56296bd4739beba3189a2879477 is 175, key is test_row_0/A:col10/1732148956108/Put/seqid=0 2024-11-21T00:29:16,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742518_1694 (size=31255) 2024-11-21T00:29:16,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6df3a56296bd4739beba3189a2879477 2024-11-21T00:29:16,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T00:29:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:16,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5bbc04f1a4f54fab8c293fbd64a5b571 is 50, key is test_row_0/B:col10/1732148956108/Put/seqid=0 2024-11-21T00:29:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742519_1695 (size=12301) 2024-11-21T00:29:16,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5bbc04f1a4f54fab8c293fbd64a5b571 2024-11-21T00:29:16,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/8b665b15c9fa4be9934444652eb4d6c4 is 50, key is test_row_0/C:col10/1732148956108/Put/seqid=0 2024-11-21T00:29:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52082 deadline: 1732149016622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,624 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_0' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742520_1696 (size=12301) 2024-11-21T00:29:16,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/8b665b15c9fa4be9934444652eb4d6c4 2024-11-21T00:29:16,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/6df3a56296bd4739beba3189a2879477 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477 2024-11-21T00:29:16,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477, entries=150, sequenceid=395, filesize=30.5 K 2024-11-21T00:29:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/5bbc04f1a4f54fab8c293fbd64a5b571 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5bbc04f1a4f54fab8c293fbd64a5b571 2024-11-21T00:29:16,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5bbc04f1a4f54fab8c293fbd64a5b571, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T00:29:16,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/8b665b15c9fa4be9934444652eb4d6c4 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/8b665b15c9fa4be9934444652eb4d6c4 2024-11-21T00:29:16,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/8b665b15c9fa4be9934444652eb4d6c4, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T00:29:16,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 619ceef852c1cdfc78ce20d02329c56f in 574ms, sequenceid=395, compaction requested=false 2024-11-21T00:29:16,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:16,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T00:29:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:16,721 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T00:29:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cbb31c5874c74dccbc589939a4746f3a_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148956130/Put/seqid=0 2024-11-21T00:29:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742521_1697 (size=12454) 2024-11-21T00:29:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:29:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:16,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149016799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149016802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149016809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149016813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149016912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149016912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149016922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149016922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149017123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149017123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149017132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149017132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,138 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cbb31c5874c74dccbc589939a4746f3a_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cbb31c5874c74dccbc589939a4746f3a_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:17,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/abc5df0362b4491fb45c0ac740a5a280, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:17,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/abc5df0362b4491fb45c0ac740a5a280 is 175, key is test_row_0/A:col10/1732148956130/Put/seqid=0 2024-11-21T00:29:17,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742522_1698 (size=31255) 2024-11-21T00:29:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:29:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149017436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149017433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149017449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149017452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,598 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/abc5df0362b4491fb45c0ac740a5a280 2024-11-21T00:29:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/805062c2815343e493b4cf7f44b73a37 is 50, key is test_row_0/B:col10/1732148956130/Put/seqid=0 2024-11-21T00:29:17,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742523_1699 (size=12301) 2024-11-21T00:29:17,654 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/805062c2815343e493b4cf7f44b73a37 2024-11-21T00:29:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/d12c4adadb0b4f939a1ad38e2523804d is 50, key is test_row_0/C:col10/1732148956130/Put/seqid=0 2024-11-21T00:29:17,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742524_1700 (size=12301) 2024-11-21T00:29:17,686 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/d12c4adadb0b4f939a1ad38e2523804d 2024-11-21T00:29:17,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/abc5df0362b4491fb45c0ac740a5a280 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280 2024-11-21T00:29:17,694 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280, entries=150, sequenceid=408, filesize=30.5 K 2024-11-21T00:29:17,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/805062c2815343e493b4cf7f44b73a37 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/805062c2815343e493b4cf7f44b73a37 2024-11-21T00:29:17,698 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/805062c2815343e493b4cf7f44b73a37, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:29:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/d12c4adadb0b4f939a1ad38e2523804d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/d12c4adadb0b4f939a1ad38e2523804d 2024-11-21T00:29:17,701 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/d12c4adadb0b4f939a1ad38e2523804d, entries=150, sequenceid=408, filesize=12.0 K 2024-11-21T00:29:17,702 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 619ceef852c1cdfc78ce20d02329c56f in 981ms, sequenceid=408, compaction requested=true 2024-11-21T00:29:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-21T00:29:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-21T00:29:17,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-21T00:29:17,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5960 sec 2024-11-21T00:29:17,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6040 sec 2024-11-21T00:29:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(8855): Flush requested on 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:17,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T00:29:17,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T00:29:17,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112144cd9682476c470f84480363487bea1c_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149017971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149017971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149017972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149017972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:17,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742525_1701 (size=12454) 2024-11-21T00:29:18,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149018082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149018082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149018084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T00:29:18,252 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:18,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T00:29:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-21T00:29:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T00:29:18,254 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T00:29:18,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T00:29:18,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T00:29:18,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149018291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149018292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149018292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,350 INFO [Thread-2619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2619. 2024-11-21T00:29:18,350 DEBUG [Thread-2619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:18,350 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,350 INFO [Thread-2611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2611. 2024-11-21T00:29:18,350 DEBUG [Thread-2611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:18,350 DEBUG [Thread-2611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,350 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:18,350 DEBUG [Thread-2611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:18,352 INFO [Thread-2613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2613. 2024-11-21T00:29:18,352 INFO [Thread-2617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2617. 2024-11-21T00:29:18,352 DEBUG [Thread-2617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:18,352 DEBUG [Thread-2613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:18,352 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,352 DEBUG [Thread-2613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,353 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:18,353 DEBUG [Thread-2613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:18,353 INFO [Thread-2615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2615. 2024-11-21T00:29:18,354 DEBUG [Thread-2615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:18,354 DEBUG [Thread-2615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,354 DEBUG [Thread-2615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:18,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T00:29:18,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:18,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112144cd9682476c470f84480363487bea1c_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112144cd9682476c470f84480363487bea1c_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:18,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/2196a149eb4e47a98b0a792953257953, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:18,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/2196a149eb4e47a98b0a792953257953 is 175, key is test_row_0/A:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:18,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742526_1702 (size=31255) 2024-11-21T00:29:18,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:18,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:18,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T00:29:18,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149018603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149018603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149018603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:18,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/2196a149eb4e47a98b0a792953257953 2024-11-21T00:29:18,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b9a979a798624dd691ed02f43621ea5d is 50, key is test_row_0/B:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:18,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742527_1703 (size=12301) 2024-11-21T00:29:18,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:18,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:18,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:18,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T00:29:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52040 deadline: 1732149018983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:18,984 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5ed4808ef0e6,38589,1732148778087 for 'test_row_1' in 619ceef852c1cdfc78ce20d02329c56f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T00:29:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:19,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:19,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:19,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:19,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52072 deadline: 1732149019113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52054 deadline: 1732149019113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:19,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T00:29:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52098 deadline: 1732149019117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 , the old value is region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T00:29:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., hostname=5ed4808ef0e6,38589,1732148778087, seqNum=5 because the exception is null or not the one we care about 2024-11-21T00:29:19,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:19,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. as already flushing 2024-11-21T00:29:19,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:19,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T00:29:19,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b9a979a798624dd691ed02f43621ea5d 2024-11-21T00:29:19,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/f042438a2b3845f98c9412952f618014 is 50, key is test_row_0/C:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:19,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742528_1704 (size=12301) 2024-11-21T00:29:19,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/f042438a2b3845f98c9412952f618014 2024-11-21T00:29:19,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/2196a149eb4e47a98b0a792953257953 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953 2024-11-21T00:29:19,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953, entries=150, sequenceid=435, filesize=30.5 K 2024-11-21T00:29:19,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/b9a979a798624dd691ed02f43621ea5d as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b9a979a798624dd691ed02f43621ea5d 2024-11-21T00:29:19,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b9a979a798624dd691ed02f43621ea5d, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T00:29:19,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/f042438a2b3845f98c9412952f618014 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/f042438a2b3845f98c9412952f618014 2024-11-21T00:29:19,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/f042438a2b3845f98c9412952f618014, entries=150, sequenceid=435, filesize=12.0 K 2024-11-21T00:29:19,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 619ceef852c1cdfc78ce20d02329c56f in 1362ms, sequenceid=435, compaction requested=true 2024-11-21T00:29:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:19,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T00:29:19,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:19,318 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:19,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T00:29:19,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:19,318 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:19,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619ceef852c1cdfc78ce20d02329c56f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T00:29:19,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:19,324 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:19,324 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/A is initiating minor compaction (all files) 2024-11-21T00:29:19,324 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/A in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,324 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/85d9f5c98e5849c3a9463a435556e953, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=122.9 K 2024-11-21T00:29:19,324 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,324 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. files: [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/85d9f5c98e5849c3a9463a435556e953, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953] 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/B is initiating minor compaction (all files) 2024-11-21T00:29:19,325 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/B in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,325 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00d575e1654c4e87802ecbf433d3ce43, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5bbc04f1a4f54fab8c293fbd64a5b571, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/805062c2815343e493b4cf7f44b73a37, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b9a979a798624dd691ed02f43621ea5d] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=48.9 K 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d9f5c98e5849c3a9463a435556e953, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148955385 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df3a56296bd4739beba3189a2879477, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732148955771 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d575e1654c4e87802ecbf433d3ce43, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148955385 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting abc5df0362b4491fb45c0ac740a5a280, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148956120 2024-11-21T00:29:19,325 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bbc04f1a4f54fab8c293fbd64a5b571, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732148955771 2024-11-21T00:29:19,326 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2196a149eb4e47a98b0a792953257953, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732148956801 2024-11-21T00:29:19,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 805062c2815343e493b4cf7f44b73a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148956120 2024-11-21T00:29:19,326 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a979a798624dd691ed02f43621ea5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732148956801 2024-11-21T00:29:19,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T00:29:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,336 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T00:29:19,338 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#B#compaction#587 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:19,338 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/d2eeddf2309a45e3ba2614bfc2f2ed01 is 50, key is test_row_0/B:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:19,339 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:19,357 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112133dea8054a3545d19f175b77158a7d25_619ceef852c1cdfc78ce20d02329c56f store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:19,359 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112133dea8054a3545d19f175b77158a7d25_619ceef852c1cdfc78ce20d02329c56f, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:19,360 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112133dea8054a3545d19f175b77158a7d25_619ceef852c1cdfc78ce20d02329c56f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b79af9d19ff74a5e841817955157da3b_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148957966/Put/seqid=0 2024-11-21T00:29:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T00:29:19,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742529_1705 (size=13289) 2024-11-21T00:29:19,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742530_1706 (size=4469) 2024-11-21T00:29:19,409 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/d2eeddf2309a45e3ba2614bfc2f2ed01 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d2eeddf2309a45e3ba2614bfc2f2ed01 2024-11-21T00:29:19,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742531_1707 (size=12454) 2024-11-21T00:29:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:19,415 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b79af9d19ff74a5e841817955157da3b_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b79af9d19ff74a5e841817955157da3b_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:19,416 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/B of 619ceef852c1cdfc78ce20d02329c56f into d2eeddf2309a45e3ba2614bfc2f2ed01(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:19,416 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:19,416 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/B, priority=12, startTime=1732148959318; duration=0sec 2024-11-21T00:29:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/61ae3058b40c4755889fae13028f55c3, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:19,416 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T00:29:19,416 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:B 2024-11-21T00:29:19,416 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T00:29:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/61ae3058b40c4755889fae13028f55c3 is 175, key is test_row_0/A:col10/1732148957966/Put/seqid=0 2024-11-21T00:29:19,418 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T00:29:19,419 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1541): 619ceef852c1cdfc78ce20d02329c56f/C is initiating minor compaction (all files) 2024-11-21T00:29:19,419 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619ceef852c1cdfc78ce20d02329c56f/C in TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:19,419 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/6e46b16cb0474d28a89ede0a1fe031b8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/8b665b15c9fa4be9934444652eb4d6c4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/d12c4adadb0b4f939a1ad38e2523804d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/f042438a2b3845f98c9412952f618014] into tmpdir=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp, totalSize=48.9 K 2024-11-21T00:29:19,419 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e46b16cb0474d28a89ede0a1fe031b8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732148955385 2024-11-21T00:29:19,419 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b665b15c9fa4be9934444652eb4d6c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732148955771 2024-11-21T00:29:19,420 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting d12c4adadb0b4f939a1ad38e2523804d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732148956120 2024-11-21T00:29:19,420 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] compactions.Compactor(225): Compacting f042438a2b3845f98c9412952f618014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732148956801 2024-11-21T00:29:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742532_1708 (size=31255) 2024-11-21T00:29:19,423 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/61ae3058b40c4755889fae13028f55c3 2024-11-21T00:29:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/d118d98347b94c25aae85101bf9f9313 is 50, key is test_row_0/B:col10/1732148957966/Put/seqid=0 2024-11-21T00:29:19,443 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#C#compaction#591 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:19,443 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/b6a5f7d0fc6e4ef9bfaa58cf48e62589 is 50, key is test_row_0/C:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742533_1709 (size=12301) 2024-11-21T00:29:19,462 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/d118d98347b94c25aae85101bf9f9313 2024-11-21T00:29:19,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/a9cd569eec1d45579f44cb16a9db842f is 50, key is test_row_0/C:col10/1732148957966/Put/seqid=0 2024-11-21T00:29:19,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742534_1710 (size=13289) 2024-11-21T00:29:19,496 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/b6a5f7d0fc6e4ef9bfaa58cf48e62589 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/b6a5f7d0fc6e4ef9bfaa58cf48e62589 2024-11-21T00:29:19,501 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/C of 619ceef852c1cdfc78ce20d02329c56f into b6a5f7d0fc6e4ef9bfaa58cf48e62589(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:19,501 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:19,501 INFO [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/C, priority=12, startTime=1732148959318; duration=0sec 2024-11-21T00:29:19,501 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:19,501 DEBUG [RS:0;5ed4808ef0e6:38589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:C 2024-11-21T00:29:19,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742535_1711 (size=12301) 2024-11-21T00:29:19,813 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619ceef852c1cdfc78ce20d02329c56f#A#compaction#588 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T00:29:19,814 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/cb87cc84b2a447a4bbf2c86ea6c80500 is 175, key is test_row_0/A:col10/1732148957954/Put/seqid=0 2024-11-21T00:29:19,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742536_1712 (size=32243) 2024-11-21T00:29:19,942 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/a9cd569eec1d45579f44cb16a9db842f 2024-11-21T00:29:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/61ae3058b40c4755889fae13028f55c3 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/61ae3058b40c4755889fae13028f55c3 2024-11-21T00:29:19,997 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/61ae3058b40c4755889fae13028f55c3, entries=150, sequenceid=444, filesize=30.5 K 2024-11-21T00:29:20,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/d118d98347b94c25aae85101bf9f9313 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d118d98347b94c25aae85101bf9f9313 2024-11-21T00:29:20,027 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d118d98347b94c25aae85101bf9f9313, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T00:29:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/a9cd569eec1d45579f44cb16a9db842f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/a9cd569eec1d45579f44cb16a9db842f 2024-11-21T00:29:20,073 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/a9cd569eec1d45579f44cb16a9db842f, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T00:29:20,080 INFO [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 619ceef852c1cdfc78ce20d02329c56f in 743ms, sequenceid=444, compaction requested=false 2024-11-21T00:29:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5ed4808ef0e6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-21T00:29:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-21T00:29:20,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-21T00:29:20,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8330 sec 2024-11-21T00:29:20,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.8380 sec 2024-11-21T00:29:20,126 INFO [Thread-2604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2604. 2024-11-21T00:29:20,126 DEBUG [Thread-2604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:20,127 DEBUG [Thread-2604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:20,127 DEBUG [Thread-2604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:20,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:20,134 INFO [Thread-2608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2608. 2024-11-21T00:29:20,134 DEBUG [Thread-2608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:20,134 DEBUG [Thread-2608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:20,134 DEBUG [Thread-2608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:20,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:20,136 INFO [Thread-2606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2606. 2024-11-21T00:29:20,136 DEBUG [Thread-2606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:20,136 DEBUG [Thread-2606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:20,136 DEBUG [Thread-2606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:20,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:20,276 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/cb87cc84b2a447a4bbf2c86ea6c80500 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb87cc84b2a447a4bbf2c86ea6c80500 2024-11-21T00:29:20,288 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619ceef852c1cdfc78ce20d02329c56f/A of 619ceef852c1cdfc78ce20d02329c56f into cb87cc84b2a447a4bbf2c86ea6c80500(size=31.5 K), total size for store is 62.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T00:29:20,288 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619ceef852c1cdfc78ce20d02329c56f: 2024-11-21T00:29:20,288 INFO [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., storeName=619ceef852c1cdfc78ce20d02329c56f/A, priority=12, startTime=1732148959318; duration=0sec 2024-11-21T00:29:20,288 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T00:29:20,288 DEBUG [RS:0;5ed4808ef0e6:38589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619ceef852c1cdfc78ce20d02329c56f:A 2024-11-21T00:29:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T00:29:20,403 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:21,024 INFO [Thread-2602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2602. 2024-11-21T00:29:21,024 DEBUG [Thread-2602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:21,024 DEBUG [Thread-2602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:21,024 DEBUG [Thread-2602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:21,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:26,663 INFO [Thread-2600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2600. 2024-11-21T00:29:26,663 DEBUG [Thread-2600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T00:29:26,664 DEBUG [Thread-2600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:26,664 DEBUG [Thread-2600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:26,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 4 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2353 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2437 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2368 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2366 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2365 2024-11-21T00:29:26,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T00:29:26,664 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:29:26,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:29:26,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:26,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:26,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:26,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T00:29:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T00:29:26,668 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148966668"}]},"ts":"1732148966668"} 2024-11-21T00:29:26,670 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T00:29:26,670 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T00:29:26,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T00:29:26,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, UNASSIGN}] 2024-11-21T00:29:26,673 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, UNASSIGN 2024-11-21T00:29:26,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=CLOSING, regionLocation=5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:26,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, UNASSIGN because future has completed 2024-11-21T00:29:26,676 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T00:29:26,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087}] 2024-11-21T00:29:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T00:29:26,840 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:26,840 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T00:29:26,840 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 619ceef852c1cdfc78ce20d02329c56f, disabling compactions & flushes 2024-11-21T00:29:26,840 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:26,840 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:26,840 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. after waiting 0 ms 2024-11-21T00:29:26,841 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:26,841 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 619ceef852c1cdfc78ce20d02329c56f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T00:29:26,856 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd45ac77407e4c17904b9db60176f3fb_619ceef852c1cdfc78ce20d02329c56f is 50, key is test_row_0/A:col10/1732148966662/Put/seqid=0 2024-11-21T00:29:26,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742537_1713 (size=12454) 2024-11-21T00:29:26,893 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T00:29:26,946 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd45ac77407e4c17904b9db60176f3fb_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd45ac77407e4c17904b9db60176f3fb_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:26,961 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/446ba2718e2d4449980c7c4dfc8742fb, store: [table=TestAcidGuarantees family=A region=619ceef852c1cdfc78ce20d02329c56f] 2024-11-21T00:29:26,961 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/446ba2718e2d4449980c7c4dfc8742fb is 175, key is test_row_0/A:col10/1732148966662/Put/seqid=0 2024-11-21T00:29:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742538_1714 (size=31255) 2024-11-21T00:29:26,981 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/446ba2718e2d4449980c7c4dfc8742fb 2024-11-21T00:29:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T00:29:27,006 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/a66a5d14ca3d4955be12c2ece5c9bf28 is 50, key is test_row_0/B:col10/1732148966662/Put/seqid=0 2024-11-21T00:29:27,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742539_1715 (size=12301) 2024-11-21T00:29:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T00:29:27,427 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/a66a5d14ca3d4955be12c2ece5c9bf28 2024-11-21T00:29:27,490 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ecc38c9b08a94bf6926e885edba54712 is 50, key is test_row_0/C:col10/1732148966662/Put/seqid=0 2024-11-21T00:29:27,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742540_1716 (size=12301) 2024-11-21T00:29:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T00:29:27,921 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ecc38c9b08a94bf6926e885edba54712 2024-11-21T00:29:27,927 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/A/446ba2718e2d4449980c7c4dfc8742fb as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/446ba2718e2d4449980c7c4dfc8742fb 2024-11-21T00:29:27,931 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/446ba2718e2d4449980c7c4dfc8742fb, entries=150, sequenceid=455, filesize=30.5 K 2024-11-21T00:29:27,931 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/B/a66a5d14ca3d4955be12c2ece5c9bf28 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/a66a5d14ca3d4955be12c2ece5c9bf28 2024-11-21T00:29:27,935 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/a66a5d14ca3d4955be12c2ece5c9bf28, entries=150, sequenceid=455, filesize=12.0 K 2024-11-21T00:29:27,941 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/.tmp/C/ecc38c9b08a94bf6926e885edba54712 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ecc38c9b08a94bf6926e885edba54712 2024-11-21T00:29:27,948 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ecc38c9b08a94bf6926e885edba54712, entries=150, sequenceid=455, filesize=12.0 K 2024-11-21T00:29:27,953 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 619ceef852c1cdfc78ce20d02329c56f in 1112ms, sequenceid=455, compaction requested=true 2024-11-21T00:29:27,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/8ac130e21deb441f9ba4ed87a398e280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/3c1a0e53e9bb40cc87eb70ada5052a3a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/aacb60c0ec5a48bc91818ede11f7daaa, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/0f927685864449f5a64e360a4245382f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/df51e1fd00c94032abb0904cc4cea60b, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f0d73bdf244748e0ae2ba89c2d280944, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/85d9f5c98e5849c3a9463a435556e953, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953] to archive 2024-11-21T00:29:27,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:29:27,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/492c8685a5574f8498fbecdfb677139b 2024-11-21T00:29:27,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6237c17453be4bdeba144eccd33706f6 2024-11-21T00:29:27,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b7c7b103476e41b2b1966f58c090d4c0 2024-11-21T00:29:27,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/8ac130e21deb441f9ba4ed87a398e280 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/8ac130e21deb441f9ba4ed87a398e280 2024-11-21T00:29:27,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1e3a0736fcbb4df1915b49183f1d1574 2024-11-21T00:29:27,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d47b8e52d6354ca883de6c85e51e058c 2024-11-21T00:29:27,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/3c1a0e53e9bb40cc87eb70ada5052a3a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/3c1a0e53e9bb40cc87eb70ada5052a3a 2024-11-21T00:29:27,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb580719d7c849a9abe4d981de5e68f7 2024-11-21T00:29:27,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f6693a1b0c2746f5a019d96bb82fba81 2024-11-21T00:29:27,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/aacb60c0ec5a48bc91818ede11f7daaa to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/aacb60c0ec5a48bc91818ede11f7daaa 2024-11-21T00:29:27,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c22d969283d94800814d7f07a5788743 2024-11-21T00:29:27,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/43424983c00e42eea085eb8ea1303378 2024-11-21T00:29:27,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/31de3cd9a6a847039beef55331e103fb 2024-11-21T00:29:27,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/0f927685864449f5a64e360a4245382f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/0f927685864449f5a64e360a4245382f 2024-11-21T00:29:27,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/c71399f2135a481e98d329279ea76fad 2024-11-21T00:29:27,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/69818aeae0024eafb18b9c14e26b600a 2024-11-21T00:29:27,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/609fe72a3b7547b8a3dac291f31e5912 2024-11-21T00:29:27,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/df51e1fd00c94032abb0904cc4cea60b to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/df51e1fd00c94032abb0904cc4cea60b 2024-11-21T00:29:27,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/4d265ff98893443ab82989fd393a3d9f 2024-11-21T00:29:27,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d25a52b5d36a48199fa0ce3263886ae9 2024-11-21T00:29:27,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/d15f17e5d865497cba815c37be6ab36a 2024-11-21T00:29:27,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f0d73bdf244748e0ae2ba89c2d280944 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/f0d73bdf244748e0ae2ba89c2d280944 2024-11-21T00:29:27,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/b2255607e1b7481ea157b46b4e5ed6d4 2024-11-21T00:29:27,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/1dd0a329d19442b1b28cf6b1311e45c5 2024-11-21T00:29:27,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/13c98726acdc4cce98789bae3428d346 2024-11-21T00:29:28,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/85d9f5c98e5849c3a9463a435556e953 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/85d9f5c98e5849c3a9463a435556e953 2024-11-21T00:29:28,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/6df3a56296bd4739beba3189a2879477 2024-11-21T00:29:28,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/abc5df0362b4491fb45c0ac740a5a280 2024-11-21T00:29:28,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/2196a149eb4e47a98b0a792953257953 2024-11-21T00:29:28,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/c43254b728d148f9a3dff0e5e020e0ed, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/dddf0562bba1420c969be57586fd58d1, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ecc1831e0c9e4c02a006050832e0d2bb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9a847013cb4e4fd1832b587742b4eacd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5ed79fa476d34176b1b0a9a3bfae86cc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b6255816e41843ba8156eab990c1e2fe, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/45120c6b64ea4124b58ae8b665d9ac4a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5dfa59d1b1604f2e828d9debdac4d577, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113994a8b68f441ea15203f412f107ce, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/514578f2b5c340edae6b6b7cfc2df391, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/bd29fe10afcc497ab826c4dd9456c22c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00cc18ae5b94492db65a2ef4ac993051, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/2f6ad0e4c2564f6c98e87283309c1188, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/7317796d282b4f15a54a6b2d59b54c86, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113c19d5e4ef4583af3ec85b9e114414, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f153ccc69b04ddcb1633194828cd8d8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/4bb14e0f7f1c4507829900d6ff90290e, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/231916760bec4027ba824785d8539eac, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ce2db34abeb342719e93e228c3cd333f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f97fccf35e14dd3856f0a6dbbb5d637, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9688189f16bb49aab28657257749b9b8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b47d8d2df4294c608fb995e05b3f44f2, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/3d129c8b46d34b5590698e55c9da8024, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/e8090b902a3645da9205d022a3475159, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00d575e1654c4e87802ecbf433d3ce43, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ebd2761412de4f43be0e67f0f05189de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5bbc04f1a4f54fab8c293fbd64a5b571, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/805062c2815343e493b4cf7f44b73a37, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b9a979a798624dd691ed02f43621ea5d] to archive 2024-11-21T00:29:28,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:29:28,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/c43254b728d148f9a3dff0e5e020e0ed to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/c43254b728d148f9a3dff0e5e020e0ed 2024-11-21T00:29:28,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/dddf0562bba1420c969be57586fd58d1 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/dddf0562bba1420c969be57586fd58d1 2024-11-21T00:29:28,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ecc1831e0c9e4c02a006050832e0d2bb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ecc1831e0c9e4c02a006050832e0d2bb 2024-11-21T00:29:28,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9a847013cb4e4fd1832b587742b4eacd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9a847013cb4e4fd1832b587742b4eacd 2024-11-21T00:29:28,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5ed79fa476d34176b1b0a9a3bfae86cc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5ed79fa476d34176b1b0a9a3bfae86cc 2024-11-21T00:29:28,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b6255816e41843ba8156eab990c1e2fe to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b6255816e41843ba8156eab990c1e2fe 2024-11-21T00:29:28,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/45120c6b64ea4124b58ae8b665d9ac4a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/45120c6b64ea4124b58ae8b665d9ac4a 2024-11-21T00:29:28,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5dfa59d1b1604f2e828d9debdac4d577 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5dfa59d1b1604f2e828d9debdac4d577 2024-11-21T00:29:28,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113994a8b68f441ea15203f412f107ce to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113994a8b68f441ea15203f412f107ce 2024-11-21T00:29:28,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/514578f2b5c340edae6b6b7cfc2df391 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/514578f2b5c340edae6b6b7cfc2df391 2024-11-21T00:29:28,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/bd29fe10afcc497ab826c4dd9456c22c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/bd29fe10afcc497ab826c4dd9456c22c 2024-11-21T00:29:28,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00cc18ae5b94492db65a2ef4ac993051 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00cc18ae5b94492db65a2ef4ac993051 2024-11-21T00:29:28,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/2f6ad0e4c2564f6c98e87283309c1188 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/2f6ad0e4c2564f6c98e87283309c1188 2024-11-21T00:29:28,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/7317796d282b4f15a54a6b2d59b54c86 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/7317796d282b4f15a54a6b2d59b54c86 2024-11-21T00:29:28,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113c19d5e4ef4583af3ec85b9e114414 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/113c19d5e4ef4583af3ec85b9e114414 2024-11-21T00:29:28,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f153ccc69b04ddcb1633194828cd8d8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f153ccc69b04ddcb1633194828cd8d8 2024-11-21T00:29:28,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/4bb14e0f7f1c4507829900d6ff90290e to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/4bb14e0f7f1c4507829900d6ff90290e 2024-11-21T00:29:28,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/231916760bec4027ba824785d8539eac to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/231916760bec4027ba824785d8539eac 2024-11-21T00:29:28,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ce2db34abeb342719e93e228c3cd333f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ce2db34abeb342719e93e228c3cd333f 2024-11-21T00:29:28,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f97fccf35e14dd3856f0a6dbbb5d637 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/6f97fccf35e14dd3856f0a6dbbb5d637 2024-11-21T00:29:28,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9688189f16bb49aab28657257749b9b8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/9688189f16bb49aab28657257749b9b8 2024-11-21T00:29:28,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b47d8d2df4294c608fb995e05b3f44f2 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b47d8d2df4294c608fb995e05b3f44f2 2024-11-21T00:29:28,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/3d129c8b46d34b5590698e55c9da8024 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/3d129c8b46d34b5590698e55c9da8024 2024-11-21T00:29:28,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/e8090b902a3645da9205d022a3475159 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/e8090b902a3645da9205d022a3475159 2024-11-21T00:29:28,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00d575e1654c4e87802ecbf433d3ce43 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/00d575e1654c4e87802ecbf433d3ce43 2024-11-21T00:29:28,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ebd2761412de4f43be0e67f0f05189de to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/ebd2761412de4f43be0e67f0f05189de 2024-11-21T00:29:28,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5bbc04f1a4f54fab8c293fbd64a5b571 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/5bbc04f1a4f54fab8c293fbd64a5b571 2024-11-21T00:29:28,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/805062c2815343e493b4cf7f44b73a37 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/805062c2815343e493b4cf7f44b73a37 2024-11-21T00:29:28,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b9a979a798624dd691ed02f43621ea5d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/b9a979a798624dd691ed02f43621ea5d 2024-11-21T00:29:28,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f8b7f3b961a477d8e61021a000c14dd, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c8683b2fbb794a0ea470b1cc46fbb44c, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/13f95eeb521743e38c80f39c843fb4de, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ea62425569f94ead8adb0186971de36a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bfb666d59e804618902c72125ae23a7f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/80cc7901a43d47a38921858a3d2c2382, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/da53ef427c4d421dbbffee944e84af31, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/418fa2266148430fb520a55a4c6a70cc, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/20ce9425bdba410baee78a2fba8afb58, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/29d78c745e8044a1a1ab4ff08f2ab17a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/670f3a62b9ab4a3faf3de84e697553f0, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/4fc6daad2cac471a8c88415a1b0e7657, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/38643824c4a343699e0174235d0639a5, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/901666994af842138e56bd68dea9d25f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/acddc336641f4357a4b8d5fc4d781ac6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/0fffb728b60f420dafd8d2fccae469a6, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bdd1c5d298874455b8d31c088c1de946, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/9b81672496314d2d91e935cb3c35fe1f, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bc2c17bb0fd04c6286a298fc102aac96, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c0a9cab015b848aeb3cd8c6829716d6a, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f93aefed5ba4a4ca74a6432e62b3eb8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ec10419847814b28b4e82d583dcd0736, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/657caa56d70b41f1b174c2eb954acb9d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/6e46b16cb0474d28a89ede0a1fe031b8, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/1424e634de344bbf86dd821f1656b4fb, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/8b665b15c9fa4be9934444652eb4d6c4, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/d12c4adadb0b4f939a1ad38e2523804d, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/f042438a2b3845f98c9412952f618014] to archive 2024-11-21T00:29:28,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T00:29:28,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f8b7f3b961a477d8e61021a000c14dd to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f8b7f3b961a477d8e61021a000c14dd 2024-11-21T00:29:28,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c8683b2fbb794a0ea470b1cc46fbb44c to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c8683b2fbb794a0ea470b1cc46fbb44c 2024-11-21T00:29:28,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/13f95eeb521743e38c80f39c843fb4de to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/13f95eeb521743e38c80f39c843fb4de 2024-11-21T00:29:28,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ea62425569f94ead8adb0186971de36a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ea62425569f94ead8adb0186971de36a 2024-11-21T00:29:28,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bfb666d59e804618902c72125ae23a7f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bfb666d59e804618902c72125ae23a7f 2024-11-21T00:29:28,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/80cc7901a43d47a38921858a3d2c2382 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/80cc7901a43d47a38921858a3d2c2382 2024-11-21T00:29:28,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/da53ef427c4d421dbbffee944e84af31 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/da53ef427c4d421dbbffee944e84af31 2024-11-21T00:29:28,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/418fa2266148430fb520a55a4c6a70cc to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/418fa2266148430fb520a55a4c6a70cc 2024-11-21T00:29:28,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/20ce9425bdba410baee78a2fba8afb58 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/20ce9425bdba410baee78a2fba8afb58 2024-11-21T00:29:28,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/29d78c745e8044a1a1ab4ff08f2ab17a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/29d78c745e8044a1a1ab4ff08f2ab17a 2024-11-21T00:29:28,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/670f3a62b9ab4a3faf3de84e697553f0 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/670f3a62b9ab4a3faf3de84e697553f0 2024-11-21T00:29:28,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/4fc6daad2cac471a8c88415a1b0e7657 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/4fc6daad2cac471a8c88415a1b0e7657 2024-11-21T00:29:28,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/38643824c4a343699e0174235d0639a5 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/38643824c4a343699e0174235d0639a5 2024-11-21T00:29:28,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/901666994af842138e56bd68dea9d25f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/901666994af842138e56bd68dea9d25f 2024-11-21T00:29:28,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/acddc336641f4357a4b8d5fc4d781ac6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/acddc336641f4357a4b8d5fc4d781ac6 2024-11-21T00:29:28,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/0fffb728b60f420dafd8d2fccae469a6 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/0fffb728b60f420dafd8d2fccae469a6 2024-11-21T00:29:28,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bdd1c5d298874455b8d31c088c1de946 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bdd1c5d298874455b8d31c088c1de946 2024-11-21T00:29:28,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/9b81672496314d2d91e935cb3c35fe1f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/9b81672496314d2d91e935cb3c35fe1f 2024-11-21T00:29:28,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bc2c17bb0fd04c6286a298fc102aac96 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/bc2c17bb0fd04c6286a298fc102aac96 2024-11-21T00:29:28,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c0a9cab015b848aeb3cd8c6829716d6a to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/c0a9cab015b848aeb3cd8c6829716d6a 2024-11-21T00:29:28,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/e403b7d1c5d54b47b7bd4fc22bb6f9a7 2024-11-21T00:29:28,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f93aefed5ba4a4ca74a6432e62b3eb8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/2f93aefed5ba4a4ca74a6432e62b3eb8 2024-11-21T00:29:28,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ec10419847814b28b4e82d583dcd0736 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ec10419847814b28b4e82d583dcd0736 2024-11-21T00:29:28,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/657caa56d70b41f1b174c2eb954acb9d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/657caa56d70b41f1b174c2eb954acb9d 2024-11-21T00:29:28,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/6e46b16cb0474d28a89ede0a1fe031b8 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/6e46b16cb0474d28a89ede0a1fe031b8 2024-11-21T00:29:28,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/1424e634de344bbf86dd821f1656b4fb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/1424e634de344bbf86dd821f1656b4fb 2024-11-21T00:29:28,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/8b665b15c9fa4be9934444652eb4d6c4 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/8b665b15c9fa4be9934444652eb4d6c4 2024-11-21T00:29:28,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/d12c4adadb0b4f939a1ad38e2523804d to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/d12c4adadb0b4f939a1ad38e2523804d 2024-11-21T00:29:28,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/f042438a2b3845f98c9412952f618014 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/f042438a2b3845f98c9412952f618014 2024-11-21T00:29:28,075 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/recovered.edits/458.seqid, newMaxSeqId=458, maxSeqId=4 2024-11-21T00:29:28,075 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. 2024-11-21T00:29:28,075 DEBUG [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 619ceef852c1cdfc78ce20d02329c56f: Waiting for close lock at 1732148966840Running coprocessor pre-close hooks at 1732148966840Disabling compacts and flushes for region at 1732148966840Disabling writes for close at 1732148966841 (+1 ms)Obtaining lock to block concurrent updates at 1732148966841Preparing flush snapshotting stores in 619ceef852c1cdfc78ce20d02329c56f at 1732148966841Finished memstore snapshotting TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732148966841Flushing stores of TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f. at 1732148966842 (+1 ms)Flushing 619ceef852c1cdfc78ce20d02329c56f/A: creating writer at 1732148966842Flushing mob file 619ceef852c1cdfc78ce20d02329c56f/A: appending metadata at 1732148966856 (+14 ms)Flushing mob file 619ceef852c1cdfc78ce20d02329c56f/A: closing flushed file at 1732148966856Flushing 619ceef852c1cdfc78ce20d02329c56f/A: appending metadata at 1732148966961 (+105 ms)Flushing 619ceef852c1cdfc78ce20d02329c56f/A: closing flushed file at 1732148966961Flushing 619ceef852c1cdfc78ce20d02329c56f/B: creating writer at 1732148966995 (+34 ms)Flushing 619ceef852c1cdfc78ce20d02329c56f/B: appending metadata at 1732148967006 (+11 ms)Flushing 619ceef852c1cdfc78ce20d02329c56f/B: closing flushed file at 1732148967006Flushing 619ceef852c1cdfc78ce20d02329c56f/C: creating writer at 1732148967478 (+472 ms)Flushing 619ceef852c1cdfc78ce20d02329c56f/C: appending metadata at 1732148967490 (+12 ms)Flushing 619ceef852c1cdfc78ce20d02329c56f/C: closing flushed file at 1732148967490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@128fd002: reopening flushed file at 1732148967927 (+437 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd05a00: reopening flushed file at 1732148967931 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55043027: reopening flushed file at 1732148967935 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 619ceef852c1cdfc78ce20d02329c56f in 1112ms, sequenceid=455, compaction requested=true at 1732148967953 (+18 ms)Writing region close event to WAL at 1732148968068 (+115 ms)Running coprocessor post-close hooks at 1732148968075 (+7 ms)Closed at 1732148968075 2024-11-21T00:29:28,077 INFO [RS_CLOSE_REGION-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=619ceef852c1cdfc78ce20d02329c56f, regionState=CLOSED 2024-11-21T00:29:28,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 because future has completed 2024-11-21T00:29:28,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-21T00:29:28,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 619ceef852c1cdfc78ce20d02329c56f, server=5ed4808ef0e6,38589,1732148778087 in 1.4030 sec 2024-11-21T00:29:28,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-21T00:29:28,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619ceef852c1cdfc78ce20d02329c56f, UNASSIGN in 1.4090 sec 2024-11-21T00:29:28,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-21T00:29:28,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4110 sec 2024-11-21T00:29:28,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732148968084"}]},"ts":"1732148968084"} 2024-11-21T00:29:28,085 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T00:29:28,085 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T00:29:28,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4210 sec 2024-11-21T00:29:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T00:29:28,812 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:28,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T00:29:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:28,815 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T00:29:28,816 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:28,832 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,845 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C, FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/recovered.edits] 2024-11-21T00:29:28,864 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/446ba2718e2d4449980c7c4dfc8742fb to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/446ba2718e2d4449980c7c4dfc8742fb 2024-11-21T00:29:28,876 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/61ae3058b40c4755889fae13028f55c3 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/61ae3058b40c4755889fae13028f55c3 2024-11-21T00:29:28,878 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb87cc84b2a447a4bbf2c86ea6c80500 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/A/cb87cc84b2a447a4bbf2c86ea6c80500 2024-11-21T00:29:28,881 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/a66a5d14ca3d4955be12c2ece5c9bf28 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/a66a5d14ca3d4955be12c2ece5c9bf28 2024-11-21T00:29:28,882 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d118d98347b94c25aae85101bf9f9313 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d118d98347b94c25aae85101bf9f9313 2024-11-21T00:29:28,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d2eeddf2309a45e3ba2614bfc2f2ed01 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/B/d2eeddf2309a45e3ba2614bfc2f2ed01 2024-11-21T00:29:28,886 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/a9cd569eec1d45579f44cb16a9db842f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/a9cd569eec1d45579f44cb16a9db842f 2024-11-21T00:29:28,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/b6a5f7d0fc6e4ef9bfaa58cf48e62589 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/b6a5f7d0fc6e4ef9bfaa58cf48e62589 2024-11-21T00:29:28,888 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ecc38c9b08a94bf6926e885edba54712 to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/C/ecc38c9b08a94bf6926e885edba54712 2024-11-21T00:29:28,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/recovered.edits/458.seqid to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f/recovered.edits/458.seqid 2024-11-21T00:29:28,891 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/default/TestAcidGuarantees/619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,891 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T00:29:28,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:29:28,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T00:29:28,896 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210bc6ddd1d4174c8bb58b01c4648df0b2_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210bc6ddd1d4174c8bb58b01c4648df0b2_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,898 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121177086f43d674c52a8cba5855bf7ee02_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121177086f43d674c52a8cba5855bf7ee02_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,900 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211aed6dfc75a148d1a8fb02c4b0850619_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211aed6dfc75a148d1a8fb02c4b0850619_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,902 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121274cbb91a9304c618c14bac19bcee0bd_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121274cbb91a9304c618c14bac19bcee0bd_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,912 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213170fd0d97b1431499c43353a5da9027_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213170fd0d97b1431499c43353a5da9027_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T00:29:28,933 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112142463b41a1db40d4a35ab779c8ce09c5_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112142463b41a1db40d4a35ab779c8ce09c5_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,934 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112144cd9682476c470f84480363487bea1c_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112144cd9682476c470f84480363487bea1c_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214ad45e710151418d952f3bb357318a86_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214ad45e710151418d952f3bb357318a86_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,938 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214b963f398d3c431f9c64e2ea4eb55b11_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214b963f398d3c431f9c64e2ea4eb55b11_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,940 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121573e836aa7af44428db1b09430ff619d_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121573e836aa7af44428db1b09430ff619d_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,942 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173cbcd246b8e4096a94500b24646f8e5_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173cbcd246b8e4096a94500b24646f8e5_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,945 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182458c2946e543b2a46352d0ed488081_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182458c2946e543b2a46352d0ed488081_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,960 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121939ec99f17654658acbb6574f153a517_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121939ec99f17654658acbb6574f153a517_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,962 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112196143cac0ef34ebc978270aaf2c79f35_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112196143cac0ef34ebc978270aaf2c79f35_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,963 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121980c06f3f1a9450cbd5074a10b310974_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121980c06f3f1a9450cbd5074a10b310974_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,965 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a1e82b239b4346af9ab727d742c5923a_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a1e82b239b4346af9ab727d742c5923a_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,966 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2de241320e24d9d96a05bcb4a504606_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2de241320e24d9d96a05bcb4a504606_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,971 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b79af9d19ff74a5e841817955157da3b_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b79af9d19ff74a5e841817955157da3b_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,985 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b82e14efade8428ba3cc57ed02f9e199_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b82e14efade8428ba3cc57ed02f9e199_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b9b68bdaf2204e0eb9eb8ff8eea399b9_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b9b68bdaf2204e0eb9eb8ff8eea399b9_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,990 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd45ac77407e4c17904b9db60176f3fb_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd45ac77407e4c17904b9db60176f3fb_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c27eee7dbcd84c5eb88dfb9d21b98d8c_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c27eee7dbcd84c5eb88dfb9d21b98d8c_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,997 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cbb31c5874c74dccbc589939a4746f3a_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cbb31c5874c74dccbc589939a4746f3a_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,999 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4329e7130b946dfbfec33973de999aa_619ceef852c1cdfc78ce20d02329c56f to hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e4329e7130b946dfbfec33973de999aa_619ceef852c1cdfc78ce20d02329c56f 2024-11-21T00:29:28,999 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T00:29:29,005 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:29,009 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T00:29:29,014 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T00:29:29,015 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:29,015 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T00:29:29,015 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732148969015"}]},"ts":"9223372036854775807"} 2024-11-21T00:29:29,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T00:29:29,018 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 619ceef852c1cdfc78ce20d02329c56f, NAME => 'TestAcidGuarantees,,1732148935933.619ceef852c1cdfc78ce20d02329c56f.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T00:29:29,018 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T00:29:29,018 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732148969018"}]},"ts":"9223372036854775807"} 2024-11-21T00:29:29,025 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T00:29:29,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T00:29:29,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 213 msec 2024-11-21T00:29:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T00:29:29,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T00:29:29,132 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T00:29:29,154 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 238), OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1150 (was 1081) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3378 (was 2131) - AvailableMemoryMB LEAK? - 2024-11-21T00:29:29,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T00:29:29,155 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:29:29,155 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:29:29,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:29,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:29,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T00:29:29,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T00:29:29,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117780854, stopped=false 2024-11-21T00:29:29,156 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5ed4808ef0e6,38537,1732148777004 2024-11-21T00:29:29,212 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T00:29:29,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T00:29:29,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:29:29,216 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:29:29,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T00:29:29,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T00:29:29,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:29:29,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:29,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:29:29,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T00:29:29,218 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5ed4808ef0e6,38589,1732148778087' ***** 2024-11-21T00:29:29,218 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T00:29:29,218 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T00:29:29,218 INFO [RS:0;5ed4808ef0e6:38589 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T00:29:29,218 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(959): stopping server 5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5ed4808ef0e6:38589. 2024-11-21T00:29:29,219 DEBUG [RS:0;5ed4808ef0e6:38589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T00:29:29,219 DEBUG [RS:0;5ed4808ef0e6:38589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T00:29:29,219 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T00:29:29,220 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T00:29:29,220 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T00:29:29,220 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:29,221 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T00:29:29,221 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T00:29:29,221 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T00:29:29,221 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T00:29:29,221 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T00:29:29,221 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T00:29:29,260 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/info/1170e7f480b7463982b3292bc5b9ea3a is 95, key is TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637./info:/1732148816036/DeleteFamily/seqid=0 2024-11-21T00:29:29,273 INFO [regionserver/5ed4808ef0e6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T00:29:29,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742541_1717 (size=6514) 2024-11-21T00:29:29,420 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:29,618 INFO [regionserver/5ed4808ef0e6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T00:29:29,618 INFO [regionserver/5ed4808ef0e6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T00:29:29,621 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:29,699 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/info/1170e7f480b7463982b3292bc5b9ea3a 2024-11-21T00:29:29,743 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/ns/94b3ddb36b1e47f0b908a8b78ad3dd47 is 93, key is TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637./ns:/1732148815990/DeleteFamily/seqid=0 2024-11-21T00:29:29,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742542_1718 (size=5906) 2024-11-21T00:29:29,821 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:30,021 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:30,177 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/ns/94b3ddb36b1e47f0b908a8b78ad3dd47 2024-11-21T00:29:30,221 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/rep_barrier/3d71d9480e2546989397c852201b4244 is 102, key is TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637./rep_barrier:/1732148815990/DeleteFamily/seqid=0 2024-11-21T00:29:30,222 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:30,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742543_1719 (size=6025) 2024-11-21T00:29:30,280 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/rep_barrier/3d71d9480e2546989397c852201b4244 2024-11-21T00:29:30,321 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/table/1cdf20f5509f44f2aeffce8796d2694f is 96, key is TestAcidGuarantees,,1732148783298.b03d35ead3c53e10e140da78ec4bd637./table:/1732148815990/DeleteFamily/seqid=0 2024-11-21T00:29:30,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742544_1720 (size=5936) 2024-11-21T00:29:30,402 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5ed4808ef0e6:38537 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-21T00:29:30,424 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:30,624 DEBUG [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T00:29:30,739 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/table/1cdf20f5509f44f2aeffce8796d2694f 2024-11-21T00:29:30,742 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/info/1170e7f480b7463982b3292bc5b9ea3a as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/info/1170e7f480b7463982b3292bc5b9ea3a 2024-11-21T00:29:30,745 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/info/1170e7f480b7463982b3292bc5b9ea3a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T00:29:30,746 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/ns/94b3ddb36b1e47f0b908a8b78ad3dd47 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/ns/94b3ddb36b1e47f0b908a8b78ad3dd47 2024-11-21T00:29:30,750 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/ns/94b3ddb36b1e47f0b908a8b78ad3dd47, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T00:29:30,750 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/rep_barrier/3d71d9480e2546989397c852201b4244 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/rep_barrier/3d71d9480e2546989397c852201b4244 2024-11-21T00:29:30,754 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/rep_barrier/3d71d9480e2546989397c852201b4244, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T00:29:30,755 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/.tmp/table/1cdf20f5509f44f2aeffce8796d2694f as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/table/1cdf20f5509f44f2aeffce8796d2694f 2024-11-21T00:29:30,758 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/table/1cdf20f5509f44f2aeffce8796d2694f, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T00:29:30,759 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1538ms, sequenceid=90, compaction requested=false 2024-11-21T00:29:30,771 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T00:29:30,772 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T00:29:30,772 INFO [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T00:29:30,772 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732148969220Running coprocessor pre-close hooks at 1732148969221 (+1 ms)Disabling compacts and flushes for region at 1732148969221Disabling writes for close at 1732148969221Obtaining lock to block concurrent updates at 1732148969221Preparing flush snapshotting stores in 1588230740 at 1732148969221Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732148969221Flushing stores of hbase:meta,,1.1588230740 at 1732148969222 (+1 ms)Flushing 1588230740/info: creating writer at 1732148969223 (+1 ms)Flushing 1588230740/info: appending metadata at 1732148969259 (+36 ms)Flushing 1588230740/info: closing flushed file at 1732148969259Flushing 1588230740/ns: creating writer at 1732148969703 (+444 ms)Flushing 1588230740/ns: appending metadata at 1732148969742 (+39 ms)Flushing 1588230740/ns: closing flushed file at 1732148969742Flushing 1588230740/rep_barrier: creating writer at 1732148970201 (+459 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732148970221 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732148970221Flushing 1588230740/table: creating writer at 1732148970291 (+70 ms)Flushing 1588230740/table: appending metadata at 1732148970321 (+30 ms)Flushing 1588230740/table: closing flushed file at 1732148970321Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bdcca21: reopening flushed file at 1732148970742 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37dae51b: reopening flushed file at 1732148970746 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44d32ecf: reopening flushed file at 1732148970750 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71b164ac: reopening flushed file at 1732148970754 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1538ms, sequenceid=90, compaction requested=false at 1732148970759 (+5 ms)Writing region close event to WAL at 1732148970768 (+9 ms)Running coprocessor post-close hooks at 1732148970772 (+4 ms)Closed at 1732148970772 2024-11-21T00:29:30,772 DEBUG [RS_CLOSE_META-regionserver/5ed4808ef0e6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T00:29:30,825 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(976): stopping server 5ed4808ef0e6,38589,1732148778087; all regions closed. 2024-11-21T00:29:30,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741834_1010 (size=25344) 2024-11-21T00:29:30,836 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/WALs/5ed4808ef0e6,38589,1732148778087/5ed4808ef0e6%2C38589%2C1732148778087.meta.1732148782769.meta not finished, retry = 0 2024-11-21T00:29:30,939 DEBUG [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/oldWALs 2024-11-21T00:29:30,940 INFO [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5ed4808ef0e6%2C38589%2C1732148778087.meta:.meta(num 1732148782769) 2024-11-21T00:29:30,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741832_1008 (size=16668395) 2024-11-21T00:29:30,946 DEBUG [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/oldWALs 2024-11-21T00:29:30,946 INFO [RS:0;5ed4808ef0e6:38589 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5ed4808ef0e6%2C38589%2C1732148778087:(num 1732148781775) 2024-11-21T00:29:30,946 DEBUG [RS:0;5ed4808ef0e6:38589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T00:29:30,946 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T00:29:30,947 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T00:29:30,947 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.ChoreService(370): Chore service for: regionserver/5ed4808ef0e6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T00:29:30,947 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T00:29:30,947 INFO [regionserver/5ed4808ef0e6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T00:29:30,948 INFO [RS:0;5ed4808ef0e6:38589 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38589 2024-11-21T00:29:31,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T00:29:31,023 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5ed4808ef0e6,38589,1732148778087 2024-11-21T00:29:31,023 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T00:29:31,023 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5ed4808ef0e6,38589,1732148778087] 2024-11-21T00:29:31,033 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5ed4808ef0e6,38589,1732148778087 already deleted, retry=false 2024-11-21T00:29:31,034 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5ed4808ef0e6,38589,1732148778087 expired; onlineServers=0 2024-11-21T00:29:31,034 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5ed4808ef0e6,38537,1732148777004' ***** 2024-11-21T00:29:31,034 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T00:29:31,034 INFO [M:0;5ed4808ef0e6:38537 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T00:29:31,034 INFO [M:0;5ed4808ef0e6:38537 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T00:29:31,034 DEBUG [M:0;5ed4808ef0e6:38537 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T00:29:31,034 DEBUG [M:0;5ed4808ef0e6:38537 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T00:29:31,034 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.large.0-1732148781454 {}] cleaner.HFileCleaner(306): Exit Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.large.0-1732148781454,5,FailOnTimeoutGroup] 2024-11-21T00:29:31,034 DEBUG [master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.small.0-1732148781468 {}] cleaner.HFileCleaner(306): Exit Thread[master/5ed4808ef0e6:0:becomeActiveMaster-HFileCleaner.small.0-1732148781468,5,FailOnTimeoutGroup] 2024-11-21T00:29:31,034 INFO [M:0;5ed4808ef0e6:38537 {}] hbase.ChoreService(370): Chore service for: master/5ed4808ef0e6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T00:29:31,035 INFO [M:0;5ed4808ef0e6:38537 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T00:29:31,035 DEBUG [M:0;5ed4808ef0e6:38537 {}] master.HMaster(1795): Stopping service threads 2024-11-21T00:29:31,035 INFO [M:0;5ed4808ef0e6:38537 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T00:29:31,035 INFO [M:0;5ed4808ef0e6:38537 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T00:29:31,035 ERROR [M:0;5ed4808ef0e6:38537 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:45671 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45671,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T00:29:31,036 INFO [M:0;5ed4808ef0e6:38537 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T00:29:31,036 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T00:29:31,036 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T00:29:31,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T00:29:31,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T00:29:31,044 DEBUG [M:0;5ed4808ef0e6:38537 {}] zookeeper.ZKUtil(347): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T00:29:31,044 WARN [M:0;5ed4808ef0e6:38537 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T00:29:31,045 INFO [M:0;5ed4808ef0e6:38537 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/.lastflushedseqids 2024-11-21T00:29:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742545_1721 (size=72) 2024-11-21T00:29:31,134 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:29:31,134 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38589-0x1015ac8c8430001, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:29:31,135 INFO [RS:0;5ed4808ef0e6:38589 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T00:29:31,135 INFO [RS:0;5ed4808ef0e6:38589 {}] regionserver.HRegionServer(1031): Exiting; stopping=5ed4808ef0e6,38589,1732148778087; zookeeper connection closed. 2024-11-21T00:29:31,140 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f4ccacc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f4ccacc 2024-11-21T00:29:31,141 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T00:29:31,465 INFO [M:0;5ed4808ef0e6:38537 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T00:29:31,465 INFO [M:0;5ed4808ef0e6:38537 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T00:29:31,465 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T00:29:31,465 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:29:31,465 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:29:31,465 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T00:29:31,465 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:29:31,465 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=764.92 KB heapSize=940.99 KB 2024-11-21T00:29:31,484 DEBUG [M:0;5ed4808ef0e6:38537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f86bc0495944bbbb5584248276718b7 is 82, key is hbase:meta,,1/info:regioninfo/1732148782896/Put/seqid=0 2024-11-21T00:29:31,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742546_1722 (size=5672) 2024-11-21T00:29:31,896 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2174 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f86bc0495944bbbb5584248276718b7 2024-11-21T00:29:31,951 DEBUG [M:0;5ed4808ef0e6:38537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9345d903cd1f4a87a8001052be3480ef is 2275, key is \x00\x00\x00\x00\x00\x00\x00V/proc:d/1732148878321/Put/seqid=0 2024-11-21T00:29:31,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742547_1723 (size=44815) 2024-11-21T00:29:32,373 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=764.37 KB at sequenceid=2174 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9345d903cd1f4a87a8001052be3480ef 2024-11-21T00:29:32,377 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9345d903cd1f4a87a8001052be3480ef 2024-11-21T00:29:32,394 DEBUG [M:0;5ed4808ef0e6:38537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/176c5b5b1d484d45aecd1dbc542f7065 is 69, key is 5ed4808ef0e6,38589,1732148778087/rs:state/1732148781507/Put/seqid=0 2024-11-21T00:29:32,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073742548_1724 (size=5156) 2024-11-21T00:29:32,809 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2174 (bloomFilter=true), to=hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/176c5b5b1d484d45aecd1dbc542f7065 2024-11-21T00:29:32,819 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f86bc0495944bbbb5584248276718b7 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4f86bc0495944bbbb5584248276718b7 2024-11-21T00:29:32,823 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4f86bc0495944bbbb5584248276718b7, entries=8, sequenceid=2174, filesize=5.5 K 2024-11-21T00:29:32,823 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9345d903cd1f4a87a8001052be3480ef as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9345d903cd1f4a87a8001052be3480ef 2024-11-21T00:29:32,826 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9345d903cd1f4a87a8001052be3480ef 2024-11-21T00:29:32,826 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9345d903cd1f4a87a8001052be3480ef, entries=174, sequenceid=2174, filesize=43.8 K 2024-11-21T00:29:32,827 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/176c5b5b1d484d45aecd1dbc542f7065 as hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/176c5b5b1d484d45aecd1dbc542f7065 2024-11-21T00:29:32,830 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45671/user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/176c5b5b1d484d45aecd1dbc542f7065, entries=1, sequenceid=2174, filesize=5.0 K 2024-11-21T00:29:32,831 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(3140): Finished flush of dataSize ~764.92 KB/783279, heapSize ~940.70 KB/963272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1366ms, sequenceid=2174, compaction requested=false 2024-11-21T00:29:32,834 INFO [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T00:29:32,834 DEBUG [M:0;5ed4808ef0e6:38537 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732148971465Disabling compacts and flushes for region at 1732148971465Disabling writes for close at 1732148971465Obtaining lock to block concurrent updates at 1732148971465Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732148971465Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=783279, getHeapSize=963512, getOffHeapSize=0, getCellsCount=2373 at 1732148971466 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732148971466Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732148971466Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732148971484 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732148971484Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732148971913 (+429 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732148971950 (+37 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732148971950Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732148972377 (+427 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732148972393 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732148972393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bf77ff8: reopening flushed file at 1732148972818 (+425 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ecd5d52: reopening flushed file at 1732148972823 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f399c8c: reopening flushed file at 1732148972827 (+4 ms)Finished flush of dataSize ~764.92 KB/783279, heapSize ~940.70 KB/963272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1366ms, sequenceid=2174, compaction requested=false at 1732148972831 (+4 ms)Writing region close event to WAL at 1732148972834 (+3 ms)Closed at 1732148972834 2024-11-21T00:29:32,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42153 is added to blk_1073741830_1006 (size=925804) 2024-11-21T00:29:32,840 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/fae6314b-8613-71a4-4fee-b67f6464fe29/MasterData/WALs/5ed4808ef0e6,38537,1732148777004/5ed4808ef0e6%2C38537%2C1732148777004.1732148780435 not finished, retry = 0 2024-11-21T00:29:32,941 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T00:29:32,941 INFO [M:0;5ed4808ef0e6:38537 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T00:29:32,942 INFO [M:0;5ed4808ef0e6:38537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38537 2024-11-21T00:29:32,942 INFO [M:0;5ed4808ef0e6:38537 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T00:29:33,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:29:33,112 INFO [M:0;5ed4808ef0e6:38537 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T00:29:33,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38537-0x1015ac8c8430000, quorum=127.0.0.1:51011, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T00:29:33,117 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T00:29:33,119 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T00:29:33,119 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T00:29:33,119 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T00:29:33,119 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/hadoop.log.dir/,STOPPED} 2024-11-21T00:29:33,121 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T00:29:33,121 WARN [BP-1879054066-172.17.0.2-1732148771700 heartbeating to localhost/127.0.0.1:45671 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T00:29:33,121 WARN [BP-1879054066-172.17.0.2-1732148771700 heartbeating to localhost/127.0.0.1:45671 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1879054066-172.17.0.2-1732148771700 (Datanode Uuid 8fa39e67-27c4-4fd4-9c72-99e352a1aa1d) service to localhost/127.0.0.1:45671 2024-11-21T00:29:33,121 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T00:29:33,124 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/data/data1/current/BP-1879054066-172.17.0.2-1732148771700 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T00:29:33,124 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/cluster_28a3ed9d-6353-3dbf-ea20-254b79224080/data/data2/current/BP-1879054066-172.17.0.2-1732148771700 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T00:29:33,125 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T00:29:33,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T00:29:33,132 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T00:29:33,132 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T00:29:33,132 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T00:29:33,132 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e85eca5-1ed9-b2c0-cee9-e41d814189e2/hadoop.log.dir/,STOPPED} 2024-11-21T00:29:33,153 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T00:29:33,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T00:29:33,597 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T00:29:33,599 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null